diff --git a/ctop/README.md b/ctop/README.md new file mode 100644 index 000000000..321707e8c --- /dev/null +++ b/ctop/README.md @@ -0,0 +1,103 @@ +# ctop + +A curses display of what every crucible upstairs on the system is +doing, built using the `up-status` DTrace probe. + +This is Similar to what `cmon dtrace` shows, but all wrapped into a +single tool that updates in the same window. DTrace needs privileges, +so ctop has to be started with them or ran as root. + +## The table + +An example output: +``` +ctop - Unix timestamp: 1790785550 + + PID SESSION DS0 DS1 DS2 NEXTJOB DELTA EXTL RECD RECN +> 2100 aaaa1111 ACT ACT ACT 30223 710 0 0 0▇▇▇▆▅▄▃▂▁▁▁▁▁▂▃▄▅ + 2101 bbbb2222 ACT ACT ACT 11600 40 0 0 0▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + 2102 cccc3333 ACT ACT ACT 11095 5 0 0 0▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +[up/down: Move | 's': Scale | 'q': Quit] scale: all * = stale (5s) [1/3] +``` + +We show one row per upstairs session, sorted by pid. +The first two columns tell us +| | | +|---|---| +| `>` | the cursor is on this row | +| `*` | nothing has been heard from this session for five seconds | + +A session that goes quiet for thirty seconds is dropped from the +table. + +The sparkline on the right is that session's recent job rate, one +column per sample, newest on the right. It takes whatever width after +we have printed all our columns. + +## Control Keys + +| key | | +|---|---| +| up, down | move the cursor | +| `s` | switch what the sparklines are measured against | +| `d` | show the selected session's history full screen, and back | +| Esc | back from the detail view | +| `q`, Ctrl-C | quit | + +## Sparkline scale + +`s` toggles how we scale every sparkline. Both start at zero and +differ in what a full height bar means: + +- `scale: all` measures every row against the busiest sample on + screen, so the rows can be compared with each other. A quiet + session next to a busy one reads as flat. +- `scale: self` measures each row against its own largest sample, so + every row fills its height and shows its shape. Nothing can be read + across rows: a session doing ten jobs a second looks like one doing + ten thousand, and a session at a steady rate draws as a solid bar + because every sample is its own maximum. + +The footer indicates which choice is in effect. + +## Detail view + +`d` gives the selected session's job rate the whole screen: + +``` + PID SESSION DS0 DS1 DS2 NEXTJOB DELTA EXTL RECD RECN + 2100 aaaa1111 ACT ACT ACT 30223 710 0 0 0 +┌ Job rate - PID 2100 - Session aaaa1111 ──────────────────────────────┐ +│997 ⢀⠤⠒⠉⠉⠑⠢⢄ ⢠⠒⠉⠉⠉⠑⠢⡀ │ +│747 ⢀⠔⠁ ⠱⡀ ⢀⠔⠁ ⠈⢢ │ +│ ⠠⠊ ⠈⢆ ⡔⠁ ⠑⡄ ⢀⠎ │ +│498 ⠈⠢⡀ ⢀⠜ ⠘⢄ ⢠⠊ │ +│ ⠑⢄ ⢀⠎ ⠣⡀ ⡰⠁ │ +│249 ⠈⢆ ⡰⠁ ⠘⢄ ⢀⠜ │ +│ ⠣⣀ ⢀⠎ ⠣⡀ ⡠⠃ │ +│0 ⠉⠢⢄⡠⠤⠒⠁ ⠈⠑⠤⣀⡠⠔⠊ │ +└ Samples: 39 | Min: 0 | Max: 997 | Avg: 505 | Current: 710 ───────────┘ +['d'/Esc: Back | 'q': Quit] +``` + +This graph is scaled to this session's own range, since there is only +one session on screen to compare. The `s` setting still applies to +the table you come back to. + +## Replaying captured output + +`--dtrace-cmd` replaces the command ctop runs. This can aid in testing +without a live system, or when replaying specific situations: + +``` +pfexec dtrace -s tools/dtrace/upstairs_raw.d > /tmp/capture.json +ctop --dtrace-cmd "cat /tmp/capture.json" +``` + +The records are read as fast as the file can be, rather than at the +rate they were produced, so the sparklines fill immediately. ctop +stays up after the command finishes so the final state can be read + +You can also use this to have ctop read in the output from a different +dtrace script, but note that it expects a specific output. diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 77704e5ee..46b5c1ac7 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -54,9 +54,23 @@ use tokio::sync::{Notify, RwLock}; /// reports why on the way out. const DEFAULT_DTRACE_CMD: &str = r#"dtrace -Z -q -x strsize=2k -n 'crucible_upstairs*:::up-status { printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); }'"#; -/// How often the display loop wakes to look for keyboard input. +/// How often the display loop wakes to look for keyboard input. This +/// only drains the input queue. The redraw is decided separately. const INPUT_POLL_INTERVAL: Duration = Duration::from_millis(50); +/// How often the table is redrawn, showing whatever has arrived since +/// the last time. +/// +/// Each upstairs reports once a second, but at whatever point in the +/// second it happens to fire, so redrawing as records land updates a +/// different few rows each time and the table never settles. Drawing +/// on a tick of our own shows every row that reported, together. +/// +/// It is also what keeps the clock and the stale marks honest: a +/// session goes stale by not reporting, so nothing else would fire to +/// mark it. +const REFRESH_INTERVAL: Duration = Duration::from_secs(1); + /// How many trailing lines of the dtrace command's stderr to keep. We /// only need enough to say why it gave up. const MAX_STDERR_LINES: usize = 5; @@ -101,6 +115,35 @@ struct Args { output: Vec, } +/// What the dtrace command is doing. +/// +/// An empty table on its own is ambiguous, it means either that no +/// upstairs is running, or that dtrace never started. Say which it is. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +enum ReaderStatus { + /// Started, but nothing has arrived yet. + #[default] + Waiting, + + /// At least one record has been read. + Running, + + /// The command is no longer running. The string says why. + Stopped(String), +} + +impl ReaderStatus { + /// What to show under the clock, or None once records are arriving + /// and the table speaks for itself. + fn line(&self) -> Option<&str> { + match self { + ReaderStatus::Waiting => Some("waiting for dtrace output..."), + ReaderStatus::Running => None, + ReaderStatus::Stopped(why) => Some(why), + } + } +} + /// What the sparklines are measured against. /// /// Both settings start at zero; they differ in what counts as a full @@ -172,13 +215,8 @@ struct CtopState { /// itself, rather than the table of every session. detail_mode: bool, - /// Set when the dtrace command is no longer running, which tells - /// the display to stop. Otherwise a dtrace that never started - /// would leave an empty screen up with no explanation. - reader_done: bool, - - /// Why the reader stopped, reported once the terminal is back. - reader_error: Option, + /// What the dtrace command is doing, reported on screen. + reader_status: ReaderStatus, } /// Run `dtrace_cmd` and record what it produces. @@ -256,6 +294,8 @@ async fn reader_loop( // This is scoped so the lock is dropped before the notify. { let mut state = state.write().await; + state.reader_status = ReaderStatus::Running; + match state.sessions.get_mut(&wrapper.status.session_id) { Some(session) => { let delta = job_id.saturating_sub(session.last_job_id); @@ -655,6 +695,7 @@ fn render_table_view( table_state: &mut TableState, now: Instant, spark_scale: SparkScale, + reader_status: &ReaderStatus, ) -> io::Result<()> { // The busiest sample on screen, which is what the Global setting // measures against. Derived from the sessions being drawn rather @@ -675,6 +716,7 @@ fn render_table_view( let chunks = Layout::default() .constraints([ Constraint::Length(1), // timestamp + Constraint::Length(1), // what dtrace is doing Constraint::Min(0), // session table Constraint::Length(1), // key help ]) @@ -684,6 +726,10 @@ fn render_table_view( Paragraph::new(format!("ctop - Unix timestamp: {timestamp}")), chunks[0], ); + f.render_widget( + Paragraph::new(reader_status.line().unwrap_or_default()), + chunks[1], + ); let selected = table_state.selected(); @@ -693,7 +739,7 @@ fn render_table_view( // Whatever width the columns do not use goes to the sparkline. let spark_width = - (chunks[1].width as usize).saturating_sub(header.chars().count()); + (chunks[2].width as usize).saturating_sub(header.chars().count()); let rows: Vec = sessions .iter() @@ -732,7 +778,7 @@ fn render_table_view( // Rendering with the state lets the table scroll itself to keep // the cursor on screen when there are more sessions than rows. - f.render_stateful_widget(table, chunks[1], table_state); + f.render_stateful_widget(table, chunks[2], table_state); // Keys on the left, where the cursor is on the right. let position = match (selected, sessions.len()) { @@ -746,7 +792,7 @@ fn render_table_view( Constraint::Min(0), Constraint::Length(position.chars().count() as u16), ]) - .split(chunks[2]); + .split(chunks[3]); f.render_widget( Paragraph::new(format!( @@ -771,13 +817,18 @@ async fn display_loop( let mut terminal = Terminal::new(CrosstermBackend::new(io::stdout()))?; let mut table_state = TableState::default(); + let mut last_draw = Instant::now(); + + // The first pass paints, so there is something on screen before + // anything has happened. + let mut needs_draw = true; + loop { // One instant for the whole frame, so every row is judged // against the same clock. let now = Instant::now(); - // This is scoped so the lock is dropped before the wait below. - { + if needs_draw { let mut state = state.write().await; // Drop sessions that stopped reporting a while ago. An @@ -824,12 +875,11 @@ async fn display_loop( &mut table_state, now, state.spark_scale, + &state.reader_status, )?, } - if state.reader_done { - return Ok(()); - } + last_draw = Instant::now(); } tokio::select! { @@ -837,14 +887,31 @@ async fn display_loop( _ = tokio::time::sleep(INPUT_POLL_INTERVAL) => {} } + // Take everything the keyboard has already queued. One key + // per pass would let a held arrow build a backlog that keeps + // scrolling after the key is released. + let mut input_pending = false; while event::poll(Duration::ZERO)? { - if let Event::Key(key_event) = event::read()? { - if is_quit(key_event) { - return Ok(()); + match event::read()? { + Event::Key(key_event) => { + if is_quit(key_event) { + return Ok(()); + } + if handle_key(key_event, &mut *state.write().await) { + input_pending = true; + } } - handle_key(key_event, &mut *state.write().await); + // ratatui resizes itself on the next draw; we only + // have to know that one is needed. + Event::Resize(..) => input_pending = true, + _ => {} } } + + // Input is answered at once so the display keeps up with the + // keyboard. Records are not: they wait for the next tick and + // are drawn together with everything else that arrived. + needs_draw = input_pending || last_draw.elapsed() >= REFRESH_INTERVAL; } } @@ -896,30 +963,31 @@ async fn main() -> Result<()> { let reader_notify = Arc::clone(¬ify); let dtrace_cmd = args.dtrace_cmd.clone(); let reader = tokio::spawn(async move { - let error = reader_loop(&dtrace_cmd, &reader_state, &reader_notify) - .await - .err() - .map(|e| format!("{e:#}")); - - let mut state = reader_state.write().await; - state.reader_error = error; - state.reader_done = true; - drop(state); + let result = + reader_loop(&dtrace_cmd, &reader_state, &reader_notify).await; + let why = match &result { + Ok(()) => "dtrace command finished".to_string(), + Err(e) => format!("{e:#}"), + }; + reader_state.write().await.reader_status = ReaderStatus::Stopped(why); reader_notify.notify_one(); + + result }); let display_result = display_task(&state, ¬ify, &args.output).await; // The reader is either finished already or about to be dropped // along with its child, so do not wait on it for long. - let _ = tokio::time::timeout(Duration::from_millis(100), reader).await; + let reader_result = + tokio::time::timeout(Duration::from_millis(100), reader).await; display_result?; - if let Some(error) = state.write().await.reader_error.take() { - bail!("{error}"); + // Nested because of the timeout and the join. + match reader_result { + Ok(Ok(Err(e))) => Err(e), + _ => Ok(()), } - - Ok(()) } diff --git a/tools/dtrace/README.md b/tools/dtrace/README.md index d0055e1b8..35f63a4e2 100644 --- a/tools/dtrace/README.md +++ b/tools/dtrace/README.md @@ -591,6 +591,11 @@ cmon takes and the columns each one produces. If the upstairs is not yet running, add the -Z flag to dtrace so it will wait to find the matching probe. +`ctop` shows the same fields in a curses display, one row per session +updated in place, with a sparkline of each session's job rate. It runs +an equivalent dtrace command itself, so it needs neither this script nor +a pipeline: `pfexec ctop`. See `ctop/README.md`. + ## tracegw.d This is a dtrace example script for counting IOs into and out of crucible from the guest. diff --git a/tools/make-nightly.sh b/tools/make-nightly.sh index 038d78a5d..294116f5c 100755 --- a/tools/make-nightly.sh +++ b/tools/make-nightly.sh @@ -15,6 +15,7 @@ git status >> nightly-info.txt tar cavf out/crucible-nightly.tar.gz \ target/release/cmon \ + target/release/ctop \ target/release/crudd \ target/release/crutest \ target/release/crucible-agent \