From 5e61cd11f0b35459ef876f9e1bb701775de0caca Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 4 Aug 2026 11:28:20 +0200 Subject: [PATCH 1/8] Visualize `dstack metrics` as sparklines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dstack metrics` printed one line of raw numbers per GPU, repeated per poll under `--watch`. Eight GPUs meant eight near-identical lines, and nothing about them showed whether a number was steady, climbing or a spike — the shape of a run was only visible by watching the command. Draw each series as a sparkline instead, over the metrics the server retains (~1h for a running job). Glyph height is the bucket's peak so a brief 100% cannot hide, colour is its mean so a card that *held* 100% reads differently from one that *touched* it, and the latest value is printed beside every sparkline — colour is never the only carrier. A labelled rule under the table gives the span, ending in `now` while the job is running and an absolute local time once it is not. Like `dstack logs`, this shows a single job: `--replica` and `--job` select one, both defaulting to 0. Previously every job of a service was printed at once, so comparing replicas now means one invocation each. Also send `limit` explicitly from the API client. The endpoint declares it as `limit: int = 1` rather than Optional, so omitting it silently capped the response at a single sample. Co-Authored-By: Claude Opus 5 (1M context) --- src/dstack/_internal/cli/commands/metrics.py | 160 ++++------- src/dstack/_internal/cli/utils/metrics.py | 250 ++++++++++++++++++ src/dstack/_internal/cli/utils/sparkline.py | 98 +++++++ src/dstack/api/server/_metrics.py | 26 +- .../_internal/cli/commands/test_metrics.py | 81 +++--- src/tests/_internal/cli/utils/test_metrics.py | 189 +++++++++++++ .../_internal/cli/utils/test_sparkline.py | 72 +++++ 7 files changed, 732 insertions(+), 144 deletions(-) create mode 100644 src/dstack/_internal/cli/utils/metrics.py create mode 100644 src/dstack/_internal/cli/utils/sparkline.py create mode 100644 src/tests/_internal/cli/utils/test_metrics.py create mode 100644 src/tests/_internal/cli/utils/test_sparkline.py diff --git a/src/dstack/_internal/cli/commands/metrics.py b/src/dstack/_internal/cli/commands/metrics.py index 16092748c8..78feea6fe6 100644 --- a/src/dstack/_internal/cli/commands/metrics.py +++ b/src/dstack/_internal/cli/commands/metrics.py @@ -1,21 +1,22 @@ import argparse import time -from typing import Any, List, Optional from rich.live import Live -from rich.table import Table from dstack._internal.cli.commands import APIBaseCommand from dstack._internal.cli.services.completion import RunNameCompleter from dstack._internal.cli.utils.common import ( - LIVE_TABLE_PROVISION_INTERVAL_SECS, LIVE_TABLE_REFRESH_RATE_PER_SEC, - add_row_from_dict, console, ) +from dstack._internal.cli.utils.metrics import ( + MAX_SAMPLES, + WATCH_INTERVAL_SECONDS, + get_metrics_table, +) from dstack._internal.core.errors import CLIError -from dstack._internal.core.models.instances import Resources from dstack._internal.core.models.metrics import JobMetrics +from dstack._internal.core.models.runs import Job from dstack.api._public import Client from dstack.api._public.runs import Run @@ -33,121 +34,64 @@ def _register(self): help="Watch run metrics in realtime", action="store_true", ) + self._parser.add_argument( + "--replica", + help="The replica number. Defaults to 0.", + type=int, + default=0, + ) + self._parser.add_argument( + "--job", + help="The job number inside the replica. Defaults to 0.", + type=int, + default=0, + ) def _command(self, args: argparse.Namespace): super()._command(args) - run = self.api.runs.get(run_name=args.run_name) - if run is None: - raise CLIError(f"Run {args.run_name} not found") - metrics = _get_run_jobs_metrics(api=self.api, run=run) + job, metrics = self._fetch(args) if not args.watch: - console.print(_get_metrics_table(run, metrics)) + console.print(get_metrics_table(job, metrics)) return try: with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live: while True: - live.update(_get_metrics_table(run, metrics)) - time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS) - run = self.api.runs.get(run_name=args.run_name) - if run is None: - raise CLIError(f"Run {args.run_name} not found") - metrics = _get_run_jobs_metrics(api=self.api, run=run) + live.update(get_metrics_table(job, metrics)) + time.sleep(WATCH_INTERVAL_SECONDS) + job, metrics = self._fetch(args) except KeyboardInterrupt: pass + def _fetch(self, args: argparse.Namespace) -> tuple[Job, JobMetrics]: + run = self.api.runs.get(run_name=args.run_name) + if run is None: + raise CLIError(f"Run {args.run_name} not found") + job = _get_job(run, args.replica, args.job) + return job, _get_job_metrics(self.api, run, job) -def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]: - metrics = [] - for job in run._run.jobs: - job_metrics = api.client.metrics.get_job_metrics( - project_name=api.project, - run_name=run.name, - replica_num=job.job_spec.replica_num, - job_num=job.job_spec.job_num, - ) - metrics.append(job_metrics) - return metrics - - -def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table: - table = Table(box=None) - table.add_column("NAME", style="bold", no_wrap=True) - table.add_column("STATUS") - table.add_column("CPU") - table.add_column("MEMORY") - table.add_column("GPU") - - run_row = {"NAME": run.name, "STATUS": run.status.value} - if len(run._run.jobs) != 1: - add_row_from_dict(table, run_row) - - for job, job_metrics in zip(run._run.jobs, metrics): - jrd = job.job_submissions[-1].job_runtime_data - jpd = job.job_submissions[-1].job_provisioning_data - resources: Optional[Resources] = None - if jrd is not None and jrd.offer is not None: - resources = jrd.offer.instance.resources - elif jpd is not None: - resources = jpd.instance_type.resources - cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent") - if cpu_usage is not None: - if resources is not None: - cpu_usage = cpu_usage / resources.cpus - cpu_usage = f"{cpu_usage:.0f}%" - memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes") - if memory_usage is not None: - memory_usage = _format_memory(memory_usage, 2) - if resources is not None: - memory_usage += f"/{_format_memory(resources.memory_mib * 1024 * 1024, 2)}" - gpu_metrics = "" - gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num") - if gpus_detected_num is not None: - for i in range(gpus_detected_num): - gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}") - gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}") - if gpu_memory_usage is not None: - if i != 0: - gpu_metrics += "\n" - gpu_metrics += f"gpu={i} mem={_format_memory(gpu_memory_usage, 2)}" - if resources is not None: - gpu_metrics += ( - f"/{_format_memory(resources.gpus[i].memory_mib * 1024 * 1024, 2)}" - ) - gpu_metrics += f" util={gpu_util_percent}%" - - job_row = { - "NAME": f" replica={job.job_spec.replica_num} job={job.job_spec.job_num}", - "STATUS": job.job_submissions[-1].status.value, - "CPU": cpu_usage or "-", - "MEMORY": memory_usage or "-", - "GPU": gpu_metrics or "-", - } - if len(run._run.jobs) == 1: - job_row.update(run_row) - add_row_from_dict(table, job_row) - - return table - - -def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]: - for metric in job_metrics.metrics: - if metric.name == name: - return metric.values[-1] - return None - - -def _format_memory(memory_bytes: int, decimal_places: int) -> str: - """See test_format_memory in tests/_internal/cli/commands/test_metrics.py for examples.""" - memory_mb = memory_bytes / 1024 / 1024 - if memory_mb >= 1024: - value = memory_mb / 1024 - unit = "GB" - else: - value = memory_mb - unit = "MB" - if decimal_places == 0: - return f"{round(value)}{unit}" - return f"{value:.{decimal_places}f}".rstrip("0").rstrip(".") + unit +def _get_job(run: Run, replica_num: int, job_num: int) -> Job: + for job in run._run.jobs: + if job.job_spec.replica_num == replica_num and job.job_spec.job_num == job_num: + return job + raise CLIError( + f"Run {run.name} has no replica={replica_num} job={job_num}." + " Use --replica and --job to select one." + ) + + +def _get_job_metrics(api: Client, run: Run, job: Job) -> JobMetrics: + """Ask for everything retained, as the UI does. + + `limit` must be sent explicitly: the endpoint declares it as `limit: int = 1`, not + Optional, so omitting it caps the response at a single sample. + """ + return api.client.metrics.get_job_metrics( + project_name=api.project, + run_name=run.name, + replica_num=job.job_spec.replica_num, + job_num=job.job_spec.job_num, + limit=MAX_SAMPLES, + ) diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py new file mode 100644 index 0000000000..9102e38504 --- /dev/null +++ b/src/dstack/_internal/cli/utils/metrics.py @@ -0,0 +1,250 @@ +"""Table rendering for `dstack metrics`.""" + +from datetime import datetime +from typing import Any, List, Optional + +from rich.console import RenderableType +from rich.table import Table +from rich.text import Text + +from dstack._internal.cli.utils.common import console +from dstack._internal.cli.utils.sparkline import ( + GPU_RAMP, + HOST_RAMP, + Ramp, + no_data, + sparkline, + supports_unicode, +) +from dstack._internal.core.models.instances import Resources +from dstack._internal.core.models.metrics import JobMetrics +from dstack._internal.core.models.runs import Job +from dstack._internal.utils.common import pretty_date + +MAX_SAMPLES = 1000 +"""How many samples to request, matching the UI. + +A sample count rather than a time window: at the server's ~10s cadence this outruns the +hour of metrics a running job retains, so the response always holds everything there is. +A fixed window would under-fill for a run younger than the window. +""" + +WATCH_INTERVAL_SECONDS = 10 +"""How often `--watch` re-fetches. + +Matched to the server's collection cadence, which is a 10s interval plus a 9s per-job +guard, so a new point cannot arrive faster than this. Not `LIVE_TABLE_PROVISION_INTERVAL_SECS` +(2s), which is short because provisioning state genuinely does change that fast. +""" + +MIN_SPARK_WIDTH = 10 +MAX_SPARK_WIDTH = 80 +_FIXED_COLUMNS = 34 +"""Columns the table needs besides the sparklines: labels, numbers and padding.""" + +_SPARKLINE_COLUMNS = 2 +"""UTILIZATION and MEMORY, reused by every row, so a cell costs two columns.""" + + +def _spark_width(console_width: int) -> int: + """How many cells fit. An hour in ten cells is six minutes each, which averages away + everything but the broadest shape, so the sparklines take whatever slack there is.""" + budget = console_width - _FIXED_COLUMNS + return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget // _SPARKLINE_COLUMNS)) + + +def get_metrics_table( + job: Job, metrics: JobMetrics, console_width: Optional[int] = None +) -> RenderableType: + """One row per resource, with `UTILIZATION` and `MEMORY` reused by all of them. + + `cpu` contributes the host's utilization and system RAM, `gpu=N` the device's + utilization and HBM: the label names the processor and the column gives that + processor's memory. Four dedicated columns instead would leave half of every row empty + and cost twice as much width per sparkline cell. + + One job, like `dstack logs` -- the endpoint is per-job, and so is the UI's metrics page. + """ + ascii_only = not supports_unicode(console) + resources = _get_resources(job) + width = _spark_width(console_width or console.width) + + table = Table(box=None) + # no header: every cell in this column already reads `cpu` or `gpu=N` + table.add_column("", style="secondary", no_wrap=True) + table.add_column("UTILIZATION", no_wrap=True) + table.add_column("MEMORY", no_wrap=True) + + table.add_row( + "cpu", + _cpu_cell(metrics, resources, width, ascii_only), + _memory_cell(metrics, resources, width, ascii_only), + ) + table.add_row("", "", "") # host and devices are different things; separate them + for index in range(_gpus_num(metrics, resources)): + table.add_row( + f"gpu={index}", + _gpu_util_cell(metrics, index, width, ascii_only), + _gpu_memory_cell(metrics, resources, index, width, ascii_only), + ) + window = _window(metrics) + if window is not None: + axis = _axis(width, *window) + table.add_row("", "", "") + table.add_row("", axis, axis) + return table + + +# ---------------------------------------------------------------------- cells + + +def _cpu_cell( + job_metrics: JobMetrics, resources: Optional[Resources], width: int, ascii_only: bool +) -> Text: + values = _metric_values(job_metrics, "cpu_usage_percent") + if not values: + return no_data() + cpus = resources.cpus if resources else None + if cpus: + values = [v / cpus for v in values] + label = f"{values[-1]:.0f}%" + if cpus: + label += f" of {cpus}" + return _cell(sparkline(values, width, HOST_RAMP, ascii_only=ascii_only), label) + + +def _memory_cell( + job_metrics: JobMetrics, resources: Optional[Resources], width: int, ascii_only: bool +) -> Text: + values = _metric_values(job_metrics, "memory_working_set_bytes") + if not values: + return no_data() + total = resources.memory_mib * 1024 * 1024 if resources else None + return _level_cell(values, total, width, ascii_only, HOST_RAMP) + + +def _gpu_memory_cell( + job_metrics: JobMetrics, + resources: Optional[Resources], + index: int, + width: int, + ascii_only: bool, +) -> Text: + values = _metric_values(job_metrics, f"gpu_memory_usage_bytes_gpu{index}") + if not values: + return no_data() + total = None + if resources and index < len(resources.gpus): + total = resources.gpus[index].memory_mib * 1024 * 1024 + return _level_cell(values, total, width, ascii_only, GPU_RAMP) + + +def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int, ascii_only: bool) -> Text: + values = _metric_values(job_metrics, f"gpu_util_percent_gpu{index}") + if not values: + return no_data() + return _cell(sparkline(values, width, GPU_RAMP, ascii_only=ascii_only), f"{values[-1]:.0f}%") + + +def _level_cell( + values: List[float], total: Optional[float], width: int, ascii_only: bool, ramp: Ramp +) -> Text: + """Memory: fraction of capacity, plus used/total in GB.""" + percents = [v / total * 100 for v in values] if total else values + label = format_memory(values[-1], 0) + if total: + label += f"/{format_memory(total, 0)}" + return _cell(sparkline(percents, width, ramp, ascii_only=ascii_only), label) + + +def _cell(spark: Text, label: str) -> Text: + """A sparkline is never the only place a value lives -- its number sits beside it. + + The number is unstyled: colouring it repeats what the sparkline's colour already says, + and there is nothing green or grey about a string like `0MB/80GB`. + """ + return Text.assemble(spark, " ", label) + + +# ----------------------------------------------------------------------- data + + +def _axis(width: int, first: datetime, last: datetime) -> Text: + """A labelled rule exactly as wide as the sparkline above it, so the two align.""" + left, right = _stamp(first), _stamp(last) + fill = max(1, width - len(left) - len(right) - 2) + return Text(f"{left} " + "─" * fill + f" {right}", style="grey42") + + +def _stamp(moment: datetime) -> str: + """`now` for the newest sample of a running job, an absolute local time otherwise. + + Collection runs every ~10s, so a running job's newest sample is always within + `pretty_date`'s `now` threshold; anything else is old enough to deserve a real time, + and a finished run then cannot be mistaken for live data. + """ + if pretty_date(moment) == "now": + return "now" + local = moment.astimezone() + return f"{local.day} {local:%b %H:%M}" + + +def _window(job_metrics: JobMetrics) -> Optional[tuple[datetime, datetime]]: + """Oldest and newest sample, or None if there are none.""" + stamps = [t for metric in job_metrics.metrics for t in metric.timestamps] + return (min(stamps), max(stamps)) if stamps else None + + +def _metric_values(job_metrics: JobMetrics, name: str) -> List[Any]: + """Values for `name`, oldest first. + + The server returns points ordered latest to earliest. Reversing here, once, at the + boundary, is what keeps every sparkline drawn downstream running left-to-right in time. + """ + for metric in job_metrics.metrics: + if metric.name == name: + return list(reversed(metric.values)) + return [] + + +def _latest(job_metrics: JobMetrics, name: str) -> Optional[Any]: + values = _metric_values(job_metrics, name) + return values[-1] if values else None + + +def _gpus_num(job_metrics: JobMetrics, resources: Optional[Resources]) -> int: + """How many device rows to print. + + Prefer the offer over the metrics: it is still known when no samples have arrived, and + when the server drops GPU metrics because the device count changed mid-window. + """ + if resources is not None and resources.gpus: + return len(resources.gpus) + detected = _latest(job_metrics, "gpus_detected_num") + return int(detected) if detected else 0 + + +def _get_resources(job: Job) -> Optional[Resources]: + submission = job.job_submissions[-1] + jrd = submission.job_runtime_data + if jrd is not None and jrd.offer is not None: + return jrd.offer.instance.resources + jpd = submission.job_provisioning_data + if jpd is not None: + return jpd.instance_type.resources + return None + + +def format_memory(memory_bytes: float, decimal_places: int) -> str: + """See test_format_memory in tests/_internal/cli/commands/test_metrics.py for examples.""" + memory_mb = memory_bytes / 1024 / 1024 + if memory_mb >= 1024: + value = memory_mb / 1024 + unit = "GB" + else: + value = memory_mb + unit = "MB" + + if decimal_places == 0: + return f"{round(value)}{unit}" + return f"{value:.{decimal_places}f}".rstrip("0").rstrip(".") + unit diff --git a/src/dstack/_internal/cli/utils/sparkline.py b/src/dstack/_internal/cli/utils/sparkline.py new file mode 100644 index 0000000000..0487320761 --- /dev/null +++ b/src/dstack/_internal/cli/utils/sparkline.py @@ -0,0 +1,98 @@ +"""Sparklines for CLI tables.""" + +from typing import List, Optional, Sequence + +from rich.console import Console +from rich.text import Text + +SPARKS = "▁▂▃▄▅▆▇" +"""Stops at 7/8 height on purpose: a full block fills its cell to the top edge, so a column +of them in consecutive rows fuses into one mass and the rows stop reading as separate +series.""" + +ASCII_SPARKS = "_.:-=+*#%@" +NO_DATA = "no data" + +Ramp = Sequence[tuple[float, str]] + +# Colour encodes scope. Device metrics share one ramp because GPU utilization and GPU memory +# are two views of the same question; host metrics get a cool one so a job row is never +# mistaken for a device row, and because the economics differ -- a host CPU at 10% is normal, +# not wasted money. +GPU_RAMP: Ramp = ((25, "grey42"), (50, "chartreuse4"), (75, "chartreuse3"), (101, "green1")) +HOST_RAMP: Ramp = ( + (50, "steel_blue3"), + (90, "deep_sky_blue3"), + (97, "dark_orange3"), + (101, "red3"), +) + + +def ramp_style(value: float, ramp: Ramp) -> str: + for threshold, style in ramp: + if value < threshold: + return style + return ramp[-1][1] + + +def supports_unicode(console: Console) -> bool: + """Whether block glyphs will survive the console's encoding.""" + try: + SPARKS.encode(console.encoding or "utf-8") + except (UnicodeEncodeError, LookupError): + return False + return True + + +def slices(values: Sequence[float], width: int) -> List[tuple[float, float]]: + """`(peak, mean)` per cell, oldest first. + + Two numbers because a cell draws both: glyph height is the peak, colour is the mean. + Height alone cannot separate a card that *touched* 100% for a few seconds from one that + *held* it for minutes -- both draw full. + + Cells cover the whole series rather than its tail: metrics arrive about every 10s, so + taking the last `width` samples would cover a couple of minutes instead of the hour the + caller asked for. The last cell is the latest sample, since the number printed beside + the sparkline is that reading and the two must not contradict each other. + """ + vals = list(values) + if width < 1: + return [] + if width == 1 or len(vals) <= width: + return [(v, v) for v in vals[-width:]] + history, out = vals[:-1], [] + for i in range(width - 1): + lo = int(i * len(history) / (width - 1)) + hi = max(lo + 1, int((i + 1) * len(history) / (width - 1))) + chunk = history[lo:hi] + out.append((max(chunk), sum(chunk) / len(chunk))) + out.append((vals[-1], vals[-1])) + return out + + +def no_data() -> Text: + """Spelled out rather than the `-` used elsewhere in the CLI: next to sparklines a bare + dash reads as a stray character, and "n/a" would be wrong -- the metric applies, it just + has not arrived yet.""" + return Text(NO_DATA, style="grey58") + + +def sparkline( + values: Optional[Sequence[float]], + width: int, + ramp: Optional[Ramp] = None, + vmax: float = 100.0, + ascii_only: bool = False, +) -> Text: + """A sparkline on a fixed 0..vmax scale, so glyph height always means the same thing as + the number beside it and rows stay comparable with each other.""" + if not values: + return no_data() + glyphs = ASCII_SPARKS if ascii_only else SPARKS + text = Text() + for peak, mean in slices(values, width): + index = int(max(0.0, min(vmax, peak)) / vmax * (len(glyphs) - 1)) + shade = max(0.0, min(vmax, mean)) / vmax * 100 + text.append(glyphs[index], style=ramp_style(shade, ramp) if ramp else "cyan") + return text diff --git a/src/dstack/api/server/_metrics.py b/src/dstack/api/server/_metrics.py index b9604461db..2a5fafc20f 100644 --- a/src/dstack/api/server/_metrics.py +++ b/src/dstack/api/server/_metrics.py @@ -1,3 +1,6 @@ +from datetime import datetime +from typing import Any, Dict, Optional + from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.core.models.metrics import JobMetrics from dstack.api.server._group import APIClientGroup @@ -10,13 +13,28 @@ def get_job_metrics( run_name: str, replica_num: int = 0, job_num: int = 0, + after: Optional[datetime] = None, + before: Optional[datetime] = None, + limit: Optional[int] = None, ) -> JobMetrics: + """ + Returns job metrics ordered from the latest sample to the earliest. + + Without `after`/`before`/`limit`, the server returns one latest sample. + """ + params: Dict[str, Any] = { + "replica_num": replica_num, + "job_num": job_num, + } + if after is not None: + params["after"] = after.isoformat() + if before is not None: + params["before"] = before.isoformat() + if limit is not None: + params["limit"] = limit resp = self._request( f"/api/project/{project_name}/metrics/job/{run_name}", method="GET", - params={ - "replica_num": replica_num, - "job_num": job_num, - }, + params=params, ) return validate_extra_ignore(JobMetrics, resp.json()) diff --git a/src/tests/_internal/cli/commands/test_metrics.py b/src/tests/_internal/cli/commands/test_metrics.py index 5e9492a4a5..1e3e59d20d 100644 --- a/src/tests/_internal/cli/commands/test_metrics.py +++ b/src/tests/_internal/cli/commands/test_metrics.py @@ -1,34 +1,51 @@ +from unittest.mock import MagicMock + import pytest -from dstack._internal.cli.commands.metrics import _format_memory - - -@pytest.mark.parametrize( - "bytes_value,decimal_places,expected", - [ - # Test MB values with different decimal places - (512 * 1024 * 1024, 0, "512MB"), # exact MB, no decimals - (512 * 1024 * 1024, 2, "512MB"), # exact MB, with decimals - (512.5 * 1024 * 1024, 0, "512MB"), # decimal MB, no decimals - (512.5 * 1024 * 1024, 2, "512.5MB"), # decimal MB, 2 decimals - (512.5 * 1024 * 1024, 3, "512.5MB"), # decimal MB, 3 decimals - (999 * 1024 * 1024, 0, "999MB"), # just under 1GB, no decimals - (999 * 1024 * 1024, 2, "999MB"), # just under 1GB, with decimals - # Test GB values with different decimal places - (1.5 * 1024 * 1024 * 1024, 0, "2GB"), # decimal GB, no decimals - (1.5 * 1024 * 1024 * 1024, 2, "1.5GB"), # decimal GB, 2 decimals - (1.5 * 1024 * 1024 * 1024, 3, "1.5GB"), # decimal GB, 3 decimals - (2 * 1024 * 1024 * 1024, 0, "2GB"), # exact GB, no decimals - (2 * 1024 * 1024 * 1024, 2, "2GB"), # exact GB, with decimals - # Test edge cases - (0, 0, "0MB"), # zero bytes, no decimals - (0, 2, "0MB"), # zero bytes, with decimals - (1023 * 1024, 0, "1MB"), # just under 1MB, no decimals - (1023 * 1024, 2, "1MB"), # just under 1MB, with decimals - (1024 * 1024 * 1024 - 1, 0, "1024MB"), # just under 1GB, no decimals - (1024 * 1024 * 1024 - 1, 2, "1024MB"), # just under 1GB, with decimals - ], -) -def test_format_memory(bytes_value: int, decimal_places: int, expected: str): - result = _format_memory(bytes_value, decimal_places) - assert result == expected +from dstack._internal.cli.commands.metrics import _get_job, _get_job_metrics +from dstack._internal.cli.utils.metrics import MAX_SAMPLES +from dstack._internal.core.errors import CLIError +from dstack._internal.core.models.metrics import JobMetrics + + +def _run(replicas: int = 1, jobs_per_replica: int = 1): + run = MagicMock() + run.name = "my-run" + run._run.jobs = [] + for replica in range(replicas): + for job_num in range(jobs_per_replica): + job = MagicMock() + job.job_spec.replica_num = replica + job.job_spec.job_num = job_num + run._run.jobs.append(job) + return run + + +class TestJobSelection: + def test_defaults_to_the_first_job_of_the_first_replica(self): + """Same shape as `dstack logs`: one job, --replica/--job to pick another.""" + job = _get_job(_run(replicas=3), replica_num=0, job_num=0) + assert (job.job_spec.replica_num, job.job_spec.job_num) == (0, 0) + + def test_selects_by_replica_and_job(self): + job = _get_job(_run(replicas=3, jobs_per_replica=2), replica_num=2, job_num=1) + assert (job.job_spec.replica_num, job.job_spec.job_num) == (2, 1) + + def test_unknown_job_is_an_error(self): + with pytest.raises(CLIError, match="replica=7"): + _get_job(_run(replicas=3), replica_num=7, job_num=0) + + +class TestMetricsRequest: + def test_limit_is_sent_explicitly(self): + """The endpoint declares `limit: int = 1`, not Optional, so omitting it caps the + response at a single sample -- which renders as one glyph however long the run has + been up, and looks plausible rather than broken.""" + api = MagicMock() + api.project = "main" + api.client.metrics.get_job_metrics.return_value = JobMetrics(metrics=[]) + run = _run() + _get_job_metrics(api, run, run._run.jobs[0]) + kwargs = api.client.metrics.get_job_metrics.call_args.kwargs + assert kwargs["limit"] == MAX_SAMPLES + assert "after" not in kwargs and "before" not in kwargs diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py new file mode 100644 index 0000000000..0c4ee8b9d4 --- /dev/null +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -0,0 +1,189 @@ +from datetime import datetime, timedelta, timezone +from typing import List +from unittest.mock import MagicMock + +import pytest +from rich.console import Console +from rich.theme import Theme + +from dstack._internal.cli.utils.metrics import ( + MAX_SPARK_WIDTH, + MIN_SPARK_WIDTH, + _axis, + _metric_values, + _spark_width, + format_memory, + get_metrics_table, +) +from dstack._internal.cli.utils.sparkline import SPARKS +from dstack._internal.core.models.metrics import JobMetrics, Metric + +GIB = 1024**3 + + +def _metric(name: str, values: List[float]) -> Metric: + """Values as the server returns them: latest first.""" + now = datetime.now(timezone.utc) + return Metric(name=name, timestamps=[now] * len(values), values=list(reversed(values))) + + +def _job_metrics( + gpus: int = 0, + cpus: int = 8, + cpu_pct: float = 50.0, + gpu_util: float = 80.0, + samples: int = 60, +) -> JobMetrics: + metrics = [ + _metric("cpu_usage_percent", [cpu_pct * cpus] * samples), + _metric("memory_working_set_bytes", [4 * GIB] * samples), + ] + for index in range(gpus): + metrics.append(_metric(f"gpu_util_percent_gpu{index}", [gpu_util] * samples)) + metrics.append(_metric(f"gpu_memory_usage_bytes_gpu{index}", [60 * GIB] * samples)) + return JobMetrics(metrics=metrics) + + +def _job(gpus: int = 0, cpus: int = 8, with_resources: bool = True): + job = MagicMock() + submission = MagicMock() + if with_resources: + resources = MagicMock() + resources.cpus, resources.memory_mib = cpus, 32 * 1024 + resources.gpus = [MagicMock(memory_mib=80 * 1024) for _ in range(gpus)] + submission.job_runtime_data.offer.instance.resources = resources + else: + submission.job_runtime_data = None + submission.job_provisioning_data = None + job.job_submissions = [submission] + return job + + +def _render(job, metrics: JobMetrics, width: int = 200) -> str: + console = Console(width=width, theme=Theme({"secondary": "grey58"}), no_color=True) + with console.capture() as capture: + console.print(get_metrics_table(job, metrics, console_width=width)) + return capture.get() + + +def _lines(output: str) -> List[str]: + return [line.rstrip() for line in output.splitlines() if line.strip()] + + +def _row(output: str, label: str) -> str: + return next(line for line in _lines(output) if line.strip().startswith(label)) + + +class TestMetricValues: + def test_reverses_server_order_to_oldest_first(self): + """The service returns points latest to earliest; everything downstream assumes + the opposite, so this is the one place the series is flipped.""" + job_metrics = JobMetrics( + metrics=[ + Metric( + name="gpu_util_percent_gpu0", + timestamps=[datetime.now(timezone.utc)] * 3, + values=[30, 20, 10], + ) + ] + ) + assert _metric_values(job_metrics, "gpu_util_percent_gpu0") == [10, 20, 30] + + def test_missing_metric_is_empty(self): + assert _metric_values(JobMetrics(metrics=[]), "cpu_usage_percent") == [] + + +class TestLayout: + def test_two_columns_reused_by_every_resource(self): + """cpu and gpu=N are both processors with a utilization and a memory, so four + dedicated columns would leave half of every row empty.""" + output = _render(_job(gpus=2), _job_metrics(gpus=2)) + assert _lines(output)[0].split() == ["UTILIZATION", "MEMORY"] + assert [ln.split()[0] for ln in _lines(output)[1:4]] == ["cpu", "gpu=0", "gpu=1"] + + def test_each_row_carries_both_of_its_numbers(self): + output = _render(_job(gpus=1), _job_metrics(gpus=1, cpu_pct=25.0, gpu_util=77.0)) + assert "25% of 8" in _row(output, "cpu") + assert "77%" in _row(output, "gpu=0") + assert "60GB/80GB" in _row(output, "gpu=0") + + +class TestTimeAxis: + def test_ends_at_now_while_the_job_is_running(self): + """Collection is every ~10s, so a running job's newest sample is always `now` -- + and a finished run's is not, which stops a stale table reading as live.""" + assert _lines(_render(_job(gpus=1), _job_metrics(gpus=1)))[-1].endswith("now") + + def test_older_samples_get_an_absolute_local_time(self): + now = datetime.now(timezone.utc) + assert _axis(40, now - timedelta(hours=2), now).plain.endswith("now") + stopped = _axis(40, now - timedelta(hours=2), now - timedelta(hours=1)).plain + assert "now" not in stopped + assert ":" in stopped # a real clock time, not an age + + def test_axis_aligns_with_the_sparklines(self): + output = _render(_job(gpus=1), _job_metrics(gpus=1)) + axis, cpu_row = _lines(output)[-1], _row(output, "cpu") + assert axis.index(axis.strip()[0]) == min(cpu_row.index(g) for g in SPARKS if g in cpu_row) + + def test_no_axis_when_there_are_no_samples(self): + assert "─" not in _render(_job(gpus=1), JobMetrics(metrics=[])) + + +class TestNoData: + def test_devices_are_listed_even_with_no_samples(self): + """The device list comes from the offer, so the shape of the run is known even + when none of its numbers are.""" + output = _render(_job(gpus=4), JobMetrics(metrics=[])) + assert sum(1 for ln in _lines(output) if "gpu=" in ln) == 4 + assert "no data" in output + + def test_measured_zero_is_distinguishable_from_missing(self): + zeros = _render(_job(gpus=1), _job_metrics(gpus=1, gpu_util=0.0, cpu_pct=0.0)) + assert "0%" in zeros + assert zeros != _render(_job(gpus=1), JobMetrics(metrics=[])) + + def test_no_resources_and_no_metrics_renders_no_device_rows(self): + assert "gpu=" not in _render(_job(with_resources=False), JobMetrics(metrics=[])) + + +class TestSparkWidth: + def test_clamped_at_both_ends(self): + assert _spark_width(20) == MIN_SPARK_WIDTH + assert _spark_width(10_000) == MAX_SPARK_WIDTH + + def test_table_fits_the_terminal(self): + for width in (80, 100, 140, 190, 240): + output = _render(_job(gpus=8), _job_metrics(gpus=8), width=width) + assert max(len(ln.rstrip()) for ln in output.splitlines()) <= width + + +@pytest.mark.parametrize( + "bytes_value,decimal_places,expected", + [ + # Test MB values with different decimal places + (512 * 1024 * 1024, 0, "512MB"), # exact MB, no decimals + (512 * 1024 * 1024, 2, "512MB"), # exact MB, with decimals + (512.5 * 1024 * 1024, 0, "512MB"), # decimal MB, no decimals + (512.5 * 1024 * 1024, 2, "512.5MB"), # decimal MB, 2 decimals + (512.5 * 1024 * 1024, 3, "512.5MB"), # decimal MB, 3 decimals + (999 * 1024 * 1024, 0, "999MB"), # just under 1GB, no decimals + (999 * 1024 * 1024, 2, "999MB"), # just under 1GB, with decimals + # Test GB values with different decimal places + (1.5 * 1024 * 1024 * 1024, 0, "2GB"), # decimal GB, no decimals + (1.5 * 1024 * 1024 * 1024, 2, "1.5GB"), # decimal GB, 2 decimals + (1.5 * 1024 * 1024 * 1024, 3, "1.5GB"), # decimal GB, 3 decimals + (2 * 1024 * 1024 * 1024, 0, "2GB"), # exact GB, no decimals + (2 * 1024 * 1024 * 1024, 2, "2GB"), # exact GB, with decimals + # Test edge cases + (0, 0, "0MB"), # zero bytes, no decimals + (0, 2, "0MB"), # zero bytes, with decimals + (1023 * 1024, 0, "1MB"), # just under 1MB, no decimals + (1023 * 1024, 2, "1MB"), # just under 1MB, with decimals + (1024 * 1024 * 1024 - 1, 0, "1024MB"), # just under 1GB, no decimals + (1024 * 1024 * 1024 - 1, 2, "1024MB"), # just under 1GB, with decimals + ], +) +def test_format_memory(bytes_value: int, decimal_places: int, expected: str): + result = format_memory(bytes_value, decimal_places) + assert result == expected diff --git a/src/tests/_internal/cli/utils/test_sparkline.py b/src/tests/_internal/cli/utils/test_sparkline.py new file mode 100644 index 0000000000..6ed547d4b4 --- /dev/null +++ b/src/tests/_internal/cli/utils/test_sparkline.py @@ -0,0 +1,72 @@ +from dstack._internal.cli.utils.sparkline import ( + GPU_RAMP, + HOST_RAMP, + NO_DATA, + SPARKS, + ramp_style, + slices, + sparkline, +) + + +class TestSlices: + def test_cells_cover_the_whole_series_not_its_tail(self): + """Samples arrive ~10s apart, so taking the last `width` of them would cover a + couple of minutes instead of the hour the caller asked for.""" + cells = slices(list(range(1000)), 10) + assert len(cells) == 10 + assert cells[0][0] < 200 # first cell is early history, not a recent sample + + def test_last_cell_is_the_latest_sample(self): + """The number printed beside the sparkline is that reading; if the last cell were + its slice's peak instead, a value that just dropped would draw tall next to a + label reading 0.""" + assert slices([79.0] * 997 + [0.0, 0.0, 0.0], 10)[-1] == (0.0, 0.0) + + def test_height_is_the_peak_and_colour_is_the_mean(self): + """Height alone cannot separate a card that touched 100% briefly from one that + held it -- both draw full.""" + peak, mean = slices([0.0] * 17 + [100.0, 0.0], 2)[0] + assert peak == 100.0 + assert mean < 10.0 + + def test_shorter_than_width_is_returned_as_is(self): + assert slices([1, 2, 3], 10) == [(1, 1), (2, 2), (3, 3)] + + +class TestSparkline: + def test_width_and_fixed_scale(self): + assert sparkline([0] * 60, 10, GPU_RAMP).plain == SPARKS[0] * 10 + assert sparkline([100] * 60, 10, GPU_RAMP).plain == SPARKS[-1] * 10 + + def test_never_fills_the_cell_to_the_top(self): + """A full block touches the cell's top edge, so a column of them in consecutive + rows fuses into one mass and eight GPU rows stop reading as separate series.""" + assert "█" not in SPARKS + assert "█" not in sparkline([100] * 60, 10, GPU_RAMP).plain + + def test_height_means_fraction_of_capacity_not_of_the_window(self): + """Fitting the axis to the window would draw 154GB of 800GB as a full bar.""" + climbing = [(10 + i * 0.145) / 800 * 100 for i in range(1000)] + assert sparkline(climbing, 10, GPU_RAMP).plain[-1] in SPARKS[:3] + + def test_measured_zero_is_not_missing_data(self): + assert sparkline([0] * 60, 10, GPU_RAMP).plain == SPARKS[0] * 10 + assert sparkline([], 10, GPU_RAMP).plain == NO_DATA + + def test_ascii_fallback(self): + text = sparkline([50] * 60, 10, GPU_RAMP, ascii_only=True).plain + assert len(text) == 10 + assert " " not in text # a blank would read as missing, not as a measured zero + + +class TestRamps: + def test_scope_is_never_ambiguous(self): + """Colour marks scope, so a job row must not share a colour with a device row.""" + for value in (0, 10, 30, 60, 85, 100): + assert ramp_style(value, GPU_RAMP) != ramp_style(value, HOST_RAMP) + + def test_the_low_end_is_a_single_colour(self): + """Each glyph is coloured by its own value, so splitting "low" across bands makes + an idle GPU draw as several colours at once.""" + assert len({ramp_style(v, GPU_RAMP) for v in (0, 1, 6, 12, 24)}) == 1 From 8833da42ad3f21d24ee08ea4c36bc44483de6443 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 4 Aug 2026 11:39:16 +0200 Subject: [PATCH 2/8] Drop a metrics layout test that restated the code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_two_columns_reused_by_every_resource` asserted the column headers read `UTILIZATION` and `MEMORY` and the row labels read `cpu`, `gpu=0`, `gpu=1` — the same literals written a few lines up in `get_metrics_table`. There is no way for the layout to break that this catches and the other tests do not: a row losing its number is covered, the axis falling out of alignment with the cells is covered, and the table overflowing the terminal is covered. All it added was a second place to edit whenever the columns change. Co-Authored-By: Claude Opus 5 (1M context) --- src/tests/_internal/cli/utils/test_metrics.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py index 0c4ee8b9d4..9a44b40884 100644 --- a/src/tests/_internal/cli/utils/test_metrics.py +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -94,13 +94,6 @@ def test_missing_metric_is_empty(self): class TestLayout: - def test_two_columns_reused_by_every_resource(self): - """cpu and gpu=N are both processors with a utilization and a memory, so four - dedicated columns would leave half of every row empty.""" - output = _render(_job(gpus=2), _job_metrics(gpus=2)) - assert _lines(output)[0].split() == ["UTILIZATION", "MEMORY"] - assert [ln.split()[0] for ln in _lines(output)[1:4]] == ["cpu", "gpu=0", "gpu=1"] - def test_each_row_carries_both_of_its_numbers(self): output = _render(_job(gpus=1), _job_metrics(gpus=1, cpu_pct=25.0, gpu_util=77.0)) assert "25% of 8" in _row(output, "cpu") From e65b70aa493440dd3328feeaeeb306fb0ac164b8 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 4 Aug 2026 11:56:39 +0200 Subject: [PATCH 3/8] Update metrics docs for the new output `concepts/metrics.md` and `guides/migration/slurm.md` both printed the old `NAME STATUS CPU MEMORY GPU` table as their example, which no longer exists. Both samples are rendered from the current code rather than written by hand. The concepts page also claimed the command shows "the most recently tracked" metrics; it now charts the last hour. Records the single-job behaviour and `--replica`/`--job` there too, next to the `dstack logs` reference it mirrors. Drops a docstring on `format_memory` pointing at its test file: the path went stale when the test moved, and the function did not carry one before it was made public. Co-Authored-By: Claude Opus 5 (1M context) --- mkdocs/docs/concepts/metrics.md | 31 +++++++++++++++-------- mkdocs/docs/guides/migration/slurm.md | 9 +++++-- src/dstack/_internal/cli/utils/metrics.py | 1 - 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/mkdocs/docs/concepts/metrics.md b/mkdocs/docs/concepts/metrics.md index 889c7bc9b7..ac9f47789b 100644 --- a/mkdocs/docs/concepts/metrics.md +++ b/mkdocs/docs/concepts/metrics.md @@ -18,27 +18,36 @@ This tab displays key CPU, memory, and GPU metrics collected during the last hou ## CLI -As an alternative to the UI, you can track real-time essential metrics via the CLI. -The `dstack metrics` command displays the most recently tracked CPU, memory, and GPU metrics. +As an alternative to the UI, you can track essential metrics via the CLI. +The `dstack metrics` command charts CPU, memory, and GPU utilization over the last hour of the +job, with the latest value beside each chart.
```shell dstack metrics gentle-mayfly-1 - NAME STATUS CPU MEMORY GPU - gentle-mayfly-1 done 0% 16.27GB/2000GB gpu=0 mem=72.48GB/80GB util=0% - gpu=1 mem=64.99GB/80GB util=0% - gpu=2 mem=580MB/80GB util=0% - gpu=3 mem=4MB/80GB util=0% - gpu=4 mem=4MB/80GB util=0% - gpu=5 mem=4MB/80GB util=0% - gpu=6 mem=4MB/80GB util=0% - gpu=7 mem=292MB/80GB util=0% + UTILIZATION MEMORY + cpu ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃ 34% of 128 ▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 589GB/960GB + + gpu=0 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 88% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=1 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 95% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=2 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 92% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=3 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 99% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=4 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 84% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=5 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 85% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=6 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 84% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=7 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▅ 80% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + + 4 Aug 10:56 ─────────── now 4 Aug 10:56 ─────────── now ```
+Like [`dstack logs`](../reference/cli/dstack/logs.md), the command shows a single job. +Use `--replica` and `--job` to select one; both default to `0`. +Pass `-w` to keep the charts updating. + ## Prometheus To enable exporting metrics to Prometheus, set the diff --git a/mkdocs/docs/guides/migration/slurm.md b/mkdocs/docs/guides/migration/slurm.md index 2791075e8d..600c6e0fdd 100644 --- a/mkdocs/docs/guides/migration/slurm.md +++ b/mkdocs/docs/guides/migration/slurm.md @@ -1475,8 +1475,13 @@ Check real-time metrics: ```shell $ dstack metrics training-job - NAME STATUS CPU MEMORY GPU - training-job running 45% 16.27GB/200GB gpu=0 mem=72.48GB/80GB util=95% + + UTILIZATION MEMORY + cpu ▄▄▄▄▃▄▄▃▄▄▃▄▄▄▄▄▄▄▄▄▄▄▄▃▄▄▃ 45% of 32 ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ 16GB/200GB + + gpu=0 ▇▇▇▇▆▇▇▆▇▇▆▇▇▇▇▇▇▇▇▇▇▇▇▆▇▇▆ 95% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + + 4 Aug 10:56 ─────────── now 4 Aug 10:56 ─────────── now ``` diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py index 9102e38504..495185a571 100644 --- a/src/dstack/_internal/cli/utils/metrics.py +++ b/src/dstack/_internal/cli/utils/metrics.py @@ -236,7 +236,6 @@ def _get_resources(job: Job) -> Optional[Resources]: def format_memory(memory_bytes: float, decimal_places: int) -> str: - """See test_format_memory in tests/_internal/cli/commands/test_metrics.py for examples.""" memory_mb = memory_bytes / 1024 / 1024 if memory_mb >= 1024: value = memory_mb / 1024 From 25e30e36c3a26337c50267c3d82848955d71eddd Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 4 Aug 2026 12:33:31 +0200 Subject: [PATCH 4/8] Update the metrics samples in the blog `agentic-orchestration.md` printed the old `NAME STATUS CPU MEMORY GPU` table, and `dstack-metrics.md` illustrated the command with a screenshot of that same output. Both now show output rendered from the current code. `dstack-metrics.md` also said the command displays metrics per job for multi-job runs. Rather than rewrite the 2024 text, this adds a note in the style the post already uses for the `dstack stats` rename. Co-Authored-By: Claude Opus 5 (1M context) --- mkdocs/blog/posts/agentic-orchestration.md | 8 ++++++-- mkdocs/blog/posts/dstack-metrics.md | 21 ++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/mkdocs/blog/posts/agentic-orchestration.md b/mkdocs/blog/posts/agentic-orchestration.md index 71ad35e876..52dc97eea8 100644 --- a/mkdocs/blog/posts/agentic-orchestration.md +++ b/mkdocs/blog/posts/agentic-orchestration.md @@ -250,8 +250,12 @@ $ dstack event --within-run train-qwen ```shell $ dstack metrics train-qwen - NAME STATUS CPU MEMORY GPU - train-qwen running 92% 118GB/200GB gpu=0 mem=71GB/80GB util=97% + UTILIZATION MEMORY + cpu ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 92% of 32 ▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 118GB/200GB + + gpu=0 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 97% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + + 4 Aug 11:02 ─────────── now 4 Aug 11:02 ─────────── now ``` diff --git a/mkdocs/blog/posts/dstack-metrics.md b/mkdocs/blog/posts/dstack-metrics.md index d3bf4ffa67..d711be15ef 100644 --- a/mkdocs/blog/posts/dstack-metrics.md +++ b/mkdocs/blog/posts/dstack-metrics.md @@ -16,7 +16,23 @@ While it's possible to use third-party monitoring tools with `dstack`, it is oft track metrics out of the box. That's why, with the latest release, `dstack` introduced [`dstack stats`](../../docs/reference/cli/dstack/metrics.md), a new CLI (and API) for monitoring container metrics, including GPU usage for `NVIDIA`, `AMD`, and other accelerators. - +
+ +```shell +$ dstack metrics llama-70b-sft + + UTILIZATION MEMORY + cpu ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃ 38% of 64 ▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 301GB/480GB + + gpu=0 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 94% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB + gpu=1 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 94% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB + gpu=2 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 98% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB + gpu=3 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 90% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB + + 4 Aug 11:33 ─────────── now 4 Aug 11:33 ─────────── now +``` + +
@@ -31,6 +47,9 @@ difference is that `dstack stats` includes GPU VRAM usage and GPU utilization pe Similar to `kubectl top`, if a run consists of multiple jobs (such as distributed training or an auto-scalable service), `dstack stats` will display metrics per job. +> Note, `dstack metrics` now shows one job at a time, like `dstack logs`. Use `--replica` and `--job` to +> choose it; both default to `0`. + !!! info "HTTP API" In addition to the `dstack stats` CLI commands, metrics can be obtained via the [`/api/project/{project_name}/metrics/job/{run_name}`](../../docs/reference/http/metrics.md) HTTP endpoint. From 3d54b7b9e7252b714eb44f80d974ded9c1027736 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 4 Aug 2026 12:35:23 +0200 Subject: [PATCH 5/8] Give the doc samples a shape worth charting Every sample rendered flat -- twenty-seven identical glyphs per row. The generator drove them with a sine of period ~19 samples while a cell buckets ~13, so each bucket's peak came out the same and the variation averaged itself away. A chart that never changes argues against the feature. Drives them from profiles slower than one cell instead: a warmup ramp while weights and data load, steady training, a checkpoint stall two thirds in where the accelerators drop to idle and the CPU picks up, and memory that allocates during warmup then holds. Co-Authored-By: Claude Opus 5 (1M context) --- mkdocs/blog/posts/agentic-orchestration.md | 6 +++--- mkdocs/blog/posts/dstack-metrics.md | 12 +++++------ mkdocs/docs/concepts/metrics.md | 24 +++++++++++----------- mkdocs/docs/guides/migration/slurm.md | 6 +++--- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/mkdocs/blog/posts/agentic-orchestration.md b/mkdocs/blog/posts/agentic-orchestration.md index 52dc97eea8..1322dd1327 100644 --- a/mkdocs/blog/posts/agentic-orchestration.md +++ b/mkdocs/blog/posts/agentic-orchestration.md @@ -251,11 +251,11 @@ $ dstack event --within-run train-qwen $ dstack metrics train-qwen UTILIZATION MEMORY - cpu ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 92% of 32 ▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 118GB/200GB + cpu ▅▄▄▆▆▆▆▆▆▆▆▆▆▆▆▆▆▅▆▆▆▆▆▆▆▆▆ 91% of 32 ▃▃▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 116GB/200GB - gpu=0 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 97% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=0 ▁▂▃▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆ 92% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 70GB/80GB - 4 Aug 11:02 ─────────── now 4 Aug 11:02 ─────────── now + 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now ``` diff --git a/mkdocs/blog/posts/dstack-metrics.md b/mkdocs/blog/posts/dstack-metrics.md index d711be15ef..49af08cb31 100644 --- a/mkdocs/blog/posts/dstack-metrics.md +++ b/mkdocs/blog/posts/dstack-metrics.md @@ -22,14 +22,14 @@ for monitoring container metrics, including GPU usage for `NVIDIA`, `AMD`, and o $ dstack metrics llama-70b-sft UTILIZATION MEMORY - cpu ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃ 38% of 64 ▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 301GB/480GB + cpu ▅▄▄▄▃▃▃▃▃▃▃▃▃▃▃▃▅▅▄▂▃▃▃▃▃▃▃ 39% of 64 ▃▃▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 297GB/480GB - gpu=0 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 94% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB - gpu=1 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 94% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB - gpu=2 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 98% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB - gpu=3 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 90% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 68GB/80GB + gpu=0 ▁▂▃▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆ 89% ▄▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 67GB/80GB + gpu=1 ▁▂▆▆▆▅▆▆▆▆▆▆▆▆▆▆▁▁▅▆▆▅▆▆▆▆▆ 84% ▄▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 67GB/80GB + gpu=2 ▁▂▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆▆ 87% ▄▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 67GB/80GB + gpu=3 ▂▃▆▅▅▅▅▅▅▆▅▅▆▆▆▆▁▅▅▅▅▅▆▅▅▅▅ 82% ▄▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 67GB/80GB - 4 Aug 11:33 ─────────── now 4 Aug 11:33 ─────────── now + 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now ``` diff --git a/mkdocs/docs/concepts/metrics.md b/mkdocs/docs/concepts/metrics.md index ac9f47789b..9f5603391c 100644 --- a/mkdocs/docs/concepts/metrics.md +++ b/mkdocs/docs/concepts/metrics.md @@ -28,18 +28,18 @@ job, with the latest value beside each chart. dstack metrics gentle-mayfly-1 UTILIZATION MEMORY - cpu ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃ 34% of 128 ▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 589GB/960GB - - gpu=0 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 88% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=1 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 95% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=2 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 92% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=3 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 99% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=4 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 84% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=5 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 85% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=6 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 84% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - gpu=7 ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▅ 80% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB - - 4 Aug 10:56 ─────────── now 4 Aug 10:56 ─────────── now + cpu ▅▄▄▄▃▃▃▃▃▃▃▃▃▃▃▃▅▅▄▃▃▃▃▃▃▃▃ 41% of 128 ▃▃▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ 581GB/960GB + + gpu=0 ▁▂▃▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆ 89% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=1 ▁▂▆▆▆▅▆▆▆▆▆▆▆▆▆▆▁▁▅▆▆▅▆▆▆▆▆ 84% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=2 ▁▂▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆▆ 87% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=3 ▂▃▆▅▅▅▅▅▅▆▅▅▆▆▆▆▁▅▅▅▅▅▆▅▅▅▅ 82% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=4 ▂▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▁▆▆▆▆▆▆▆▆▆▆ 90% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=5 ▂▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆▆▆ 85% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=6 ▃▆▆▅▅▆▆▆▆▆▆▆▆▆▆▁▅▅▅▅▆▆▆▅▅▅▅ 83% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + gpu=7 ▃▆▆▆▆▆▆▆▆▆▆▆▆▆▁▁▆▆▆▆▆▆▆▆▆▆▆ 88% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB + + 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now ``` diff --git a/mkdocs/docs/guides/migration/slurm.md b/mkdocs/docs/guides/migration/slurm.md index 600c6e0fdd..e7c5fb6ac3 100644 --- a/mkdocs/docs/guides/migration/slurm.md +++ b/mkdocs/docs/guides/migration/slurm.md @@ -1477,11 +1477,11 @@ Check real-time metrics: $ dstack metrics training-job UTILIZATION MEMORY - cpu ▄▄▄▄▃▄▄▃▄▄▃▄▄▄▄▄▄▄▄▄▄▄▄▃▄▄▃ 45% of 32 ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ 16GB/200GB + cpu ▅▄▄▄▃▃▃▃▃▃▃▃▃▃▃▃▅▅▄▃▃▃▃▃▃▃▃ 45% of 32 ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ 16GB/200GB - gpu=0 ▇▇▇▇▆▇▇▆▇▇▆▇▇▇▇▇▇▇▇▇▇▇▇▆▇▇▆ 95% ▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 72GB/80GB + gpu=0 ▁▂▃▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆ 90% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB - 4 Aug 10:56 ─────────── now 4 Aug 10:56 ─────────── now + 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now ``` From c4a8d0c8d5fb3c6f76afa11226828bb9745127d6 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 4 Aug 2026 15:11:24 +0200 Subject: [PATCH 6/8] Stop the time axis where the data stops `slices` draws one cell per sample and never invents more, so a run younger than the terminal is wide fills only part of the row. `_axis` drew the full requested width regardless, which claimed a span nothing had been measured over -- and because the axis cell was then wider than the sparkline cell, Rich widened the whole UTILIZATION column to fit it, pushing MEMORY sideways and opening a gap after every label. On a nine-minute-old run at 200 columns that was 53 cells of data under a 70-column rule. Runs older than about fifteen minutes were unaffected, which is why it went unnoticed: every fixture had more samples than cells. Draws the rule dotted rather than solid while here. It only has to bind each pair of stamps -- with plain whitespace the gap inside a pair grows wider than the gap between the two columns past ~120 columns, so the stamps group with the wrong neighbour -- and it need not compete with the sparklines above it. Co-Authored-By: Claude Opus 5 (1M context) --- mkdocs/blog/posts/agentic-orchestration.md | 2 +- mkdocs/blog/posts/dstack-metrics.md | 2 +- mkdocs/docs/concepts/metrics.md | 2 +- mkdocs/docs/guides/migration/slurm.md | 2 +- src/dstack/_internal/cli/utils/metrics.py | 27 ++++++++++++++++--- src/tests/_internal/cli/utils/test_metrics.py | 16 ++++++++++- 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/mkdocs/blog/posts/agentic-orchestration.md b/mkdocs/blog/posts/agentic-orchestration.md index 1322dd1327..15e2c1b8e0 100644 --- a/mkdocs/blog/posts/agentic-orchestration.md +++ b/mkdocs/blog/posts/agentic-orchestration.md @@ -255,7 +255,7 @@ $ dstack metrics train-qwen gpu=0 ▁▂▃▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆ 92% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 70GB/80GB - 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now + 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now ``` diff --git a/mkdocs/blog/posts/dstack-metrics.md b/mkdocs/blog/posts/dstack-metrics.md index 49af08cb31..eeccdc45bc 100644 --- a/mkdocs/blog/posts/dstack-metrics.md +++ b/mkdocs/blog/posts/dstack-metrics.md @@ -29,7 +29,7 @@ $ dstack metrics llama-70b-sft gpu=2 ▁▂▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆▆ 87% ▄▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 67GB/80GB gpu=3 ▂▃▆▅▅▅▅▅▅▆▅▅▆▆▆▆▁▅▅▅▅▅▆▅▅▅▅ 82% ▄▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 67GB/80GB - 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now + 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now ``` diff --git a/mkdocs/docs/concepts/metrics.md b/mkdocs/docs/concepts/metrics.md index 9f5603391c..8e916bd987 100644 --- a/mkdocs/docs/concepts/metrics.md +++ b/mkdocs/docs/concepts/metrics.md @@ -39,7 +39,7 @@ dstack metrics gentle-mayfly-1 gpu=6 ▃▆▆▅▅▆▆▆▆▆▆▆▆▆▆▁▅▅▅▅▆▆▆▅▅▅▅ 83% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB gpu=7 ▃▆▆▆▆▆▆▆▆▆▆▆▆▆▁▁▆▆▆▆▆▆▆▆▆▆▆ 88% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB - 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now + 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now ``` diff --git a/mkdocs/docs/guides/migration/slurm.md b/mkdocs/docs/guides/migration/slurm.md index e7c5fb6ac3..fbe2dc3c35 100644 --- a/mkdocs/docs/guides/migration/slurm.md +++ b/mkdocs/docs/guides/migration/slurm.md @@ -1481,7 +1481,7 @@ $ dstack metrics training-job gpu=0 ▁▂▃▆▆▆▆▆▆▆▆▆▆▆▆▆▆▁▆▆▆▆▆▆▆▆▆ 90% ▄▅▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆▆ 71GB/80GB - 4 Aug 11:35 ─────────── now 4 Aug 11:35 ─────────── now + 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now 4 Aug 14:10 ┄┄┄┄┄┄┄┄┄┄┄ now ``` diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py index 495185a571..6f675b258a 100644 --- a/src/dstack/_internal/cli/utils/metrics.py +++ b/src/dstack/_internal/cli/utils/metrics.py @@ -39,6 +39,10 @@ MIN_SPARK_WIDTH = 10 MAX_SPARK_WIDTH = 80 + +AXIS_RULE = "┄" +"""Dotted, not solid: it only has to bind each pair of stamps, not compete with the +sparklines above it. Not whitespace -- see `_axis`.""" _FIXED_COLUMNS = 34 """Columns the table needs besides the sparklines: labels, numbers and padding.""" @@ -89,7 +93,7 @@ def get_metrics_table( ) window = _window(metrics) if window is not None: - axis = _axis(width, *window) + axis = _axis(min(width, _samples_num(metrics)), *window) table.add_row("", "", "") table.add_row("", axis, axis) return table @@ -170,10 +174,16 @@ def _cell(spark: Text, label: str) -> Text: def _axis(width: int, first: datetime, last: datetime) -> Text: - """A labelled rule exactly as wide as the sparkline above it, so the two align.""" + """A labelled rule exactly as wide as the sparkline above it, so the two align. + + The rule earns its ink by grouping. Both columns print a pair of stamps, and past about + 120 columns the whitespace inside a pair grows wider than the gap between the columns -- + 74 against 12 at width 200 -- so with nothing drawn the stamps pair up with the wrong + neighbour and the row reads as four unrelated times. + """ left, right = _stamp(first), _stamp(last) fill = max(1, width - len(left) - len(right) - 2) - return Text(f"{left} " + "─" * fill + f" {right}", style="grey42") + return Text(f"{left} " + AXIS_RULE * fill + f" {right}", style="grey42") def _stamp(moment: datetime) -> str: @@ -195,6 +205,17 @@ def _window(job_metrics: JobMetrics) -> Optional[tuple[datetime, datetime]]: return (min(stamps), max(stamps)) if stamps else None +def _samples_num(job_metrics: JobMetrics) -> int: + """How many cells the sparklines will actually occupy. + + `slices` never draws more cells than it has samples -- inventing the rest would be + fabricating data -- so a run younger than the terminal is wide fills only part of the + row. The axis has to stop where the data stops, or it claims a span nothing was + measured over and Rich widens the column to fit it. + """ + return max((len(metric.timestamps) for metric in job_metrics.metrics), default=0) + + def _metric_values(job_metrics: JobMetrics, name: str) -> List[Any]: """Values for `name`, oldest first. diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py index 9a44b40884..e45537df78 100644 --- a/src/tests/_internal/cli/utils/test_metrics.py +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timedelta, timezone from typing import List from unittest.mock import MagicMock @@ -7,6 +8,7 @@ from rich.theme import Theme from dstack._internal.cli.utils.metrics import ( + AXIS_RULE, MAX_SPARK_WIDTH, MIN_SPARK_WIDTH, _axis, @@ -120,7 +122,19 @@ def test_axis_aligns_with_the_sparklines(self): assert axis.index(axis.strip()[0]) == min(cpu_row.index(g) for g in SPARKS if g in cpu_row) def test_no_axis_when_there_are_no_samples(self): - assert "─" not in _render(_job(gpus=1), JobMetrics(metrics=[])) + assert AXIS_RULE not in _render(_job(gpus=1), JobMetrics(metrics=[])) + + def test_axis_stops_where_a_young_run_stops(self): + """The sparkline draws one cell per sample and will not invent more, so a run + younger than the terminal is wide fills only part of the row. An axis drawn to the + requested width would claim a span nothing was measured over -- and Rich would + widen the column to fit it, pushing MEMORY sideways.""" + output = _render(_job(gpus=1), _job_metrics(gpus=1, samples=12), width=200) + axis, cpu_row = _lines(output)[-1], _row(output, "cpu") + drawn = sum(1 for char in cpu_row if char in SPARKS) // 2 # two columns per row + assert drawn == 12 # one cell per sample, not the 80 cells 200 columns would allow + # the two columns print the same axis, separated by the table's own padding + assert [len(segment) for segment in re.split(r"\s{3,}", axis.strip())] == [drawn, drawn] class TestNoData: From 07d3eec97c7f37d5cbd45959dd8130cc06788b7b Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 5 Aug 2026 13:27:21 +0200 Subject: [PATCH 7/8] Cut the docstrings that only restate the code The two new modules were about a quarter prose. Most of it narrated what the line below already said -- a module docstring repeating the filename, `get_metrics_table` explaining that it builds three columns, `_gpus_num` describing its own `if`. Keeps only what the code cannot say: why `MAX_SAMPLES` is a sample count rather than a time window, why the glyph set stops short of a full block, which half of `slices`' pair is height and which is colour, that the server sends points newest-first, and that the endpoint's `limit` defaults to 1 rather than to everything. `_axis` keeps its reasoning but with measured numbers -- the previous text said the stamps misgroup "past ~120 columns" at "74 against 12", both of which were eyeballed from a render taken before the axis-width fix. Measured, the crossover is 88 columns and the gaps at 200 are 66 against 13. Co-Authored-By: Claude Opus 5 (1M context) --- src/dstack/_internal/cli/commands/metrics.py | 7 +- src/dstack/_internal/cli/utils/metrics.py | 86 ++++--------------- src/dstack/_internal/cli/utils/sparkline.py | 33 ++----- .../_internal/cli/commands/test_metrics.py | 4 - src/tests/_internal/cli/utils/test_metrics.py | 6 -- .../_internal/cli/utils/test_sparkline.py | 5 -- 6 files changed, 27 insertions(+), 114 deletions(-) diff --git a/src/dstack/_internal/cli/commands/metrics.py b/src/dstack/_internal/cli/commands/metrics.py index 78feea6fe6..3166640df7 100644 --- a/src/dstack/_internal/cli/commands/metrics.py +++ b/src/dstack/_internal/cli/commands/metrics.py @@ -83,11 +83,8 @@ def _get_job(run: Run, replica_num: int, job_num: int) -> Job: def _get_job_metrics(api: Client, run: Run, job: Job) -> JobMetrics: - """Ask for everything retained, as the UI does. - - `limit` must be sent explicitly: the endpoint declares it as `limit: int = 1`, not - Optional, so omitting it caps the response at a single sample. - """ + """`limit` must be sent explicitly: the endpoint declares it `limit: int = 1`, not + Optional, so omitting it caps the response at one sample.""" return api.client.metrics.get_job_metrics( project_name=api.project, run_name=run.name, diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py index 6f675b258a..0cfba7532d 100644 --- a/src/dstack/_internal/cli/utils/metrics.py +++ b/src/dstack/_internal/cli/utils/metrics.py @@ -1,5 +1,3 @@ -"""Table rendering for `dstack metrics`.""" - from datetime import datetime from typing import Any, List, Optional @@ -22,37 +20,25 @@ from dstack._internal.utils.common import pretty_date MAX_SAMPLES = 1000 -"""How many samples to request, matching the UI. - -A sample count rather than a time window: at the server's ~10s cadence this outruns the -hour of metrics a running job retains, so the response always holds everything there is. -A fixed window would under-fill for a run younger than the window. -""" +"""A sample count, not a window: outruns the hour a running job retains, so a young run is +never under-filled. Matches the UI.""" WATCH_INTERVAL_SECONDS = 10 -"""How often `--watch` re-fetches. - -Matched to the server's collection cadence, which is a 10s interval plus a 9s per-job -guard, so a new point cannot arrive faster than this. Not `LIVE_TABLE_PROVISION_INTERVAL_SECS` -(2s), which is short because provisioning state genuinely does change that fast. -""" +"""Matched to the server's collection cadence; a new point cannot arrive faster.""" MIN_SPARK_WIDTH = 10 MAX_SPARK_WIDTH = 80 AXIS_RULE = "┄" -"""Dotted, not solid: it only has to bind each pair of stamps, not compete with the -sparklines above it. Not whitespace -- see `_axis`.""" _FIXED_COLUMNS = 34 -"""Columns the table needs besides the sparklines: labels, numbers and padding.""" +"""Labels, numbers and padding. Hand-measured against a `589GB/1480GB`-sized label; a +wider one (a 2000GB host prints `1218GB/2000GB`) overflows and Rich ellipsizes the row +labels rather than shrinking the sparklines.""" _SPARKLINE_COLUMNS = 2 -"""UTILIZATION and MEMORY, reused by every row, so a cell costs two columns.""" def _spark_width(console_width: int) -> int: - """How many cells fit. An hour in ten cells is six minutes each, which averages away - everything but the broadest shape, so the sparklines take whatever slack there is.""" budget = console_width - _FIXED_COLUMNS return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget // _SPARKLINE_COLUMNS)) @@ -60,15 +46,6 @@ def _spark_width(console_width: int) -> int: def get_metrics_table( job: Job, metrics: JobMetrics, console_width: Optional[int] = None ) -> RenderableType: - """One row per resource, with `UTILIZATION` and `MEMORY` reused by all of them. - - `cpu` contributes the host's utilization and system RAM, `gpu=N` the device's - utilization and HBM: the label names the processor and the column gives that - processor's memory. Four dedicated columns instead would leave half of every row empty - and cost twice as much width per sparkline cell. - - One job, like `dstack logs` -- the endpoint is per-job, and so is the UI's metrics page. - """ ascii_only = not supports_unicode(console) resources = _get_resources(job) width = _spark_width(console_width or console.width) @@ -99,9 +76,6 @@ def get_metrics_table( return table -# ---------------------------------------------------------------------- cells - - def _cpu_cell( job_metrics: JobMetrics, resources: Optional[Resources], width: int, ascii_only: bool ) -> Text: @@ -153,7 +127,6 @@ def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int, ascii_only: def _level_cell( values: List[float], total: Optional[float], width: int, ascii_only: bool, ramp: Ramp ) -> Text: - """Memory: fraction of capacity, plus used/total in GB.""" percents = [v / total * 100 for v in values] if total else values label = format_memory(values[-1], 0) if total: @@ -162,24 +135,17 @@ def _level_cell( def _cell(spark: Text, label: str) -> Text: - """A sparkline is never the only place a value lives -- its number sits beside it. - - The number is unstyled: colouring it repeats what the sparkline's colour already says, - and there is nothing green or grey about a string like `0MB/80GB`. - """ return Text.assemble(spark, " ", label) -# ----------------------------------------------------------------------- data - - def _axis(width: int, first: datetime, last: datetime) -> Text: - """A labelled rule exactly as wide as the sparkline above it, so the two align. + """` ┄┄┄ `, as wide as the sparkline above it. - The rule earns its ink by grouping. Both columns print a pair of stamps, and past about - 120 columns the whitespace inside a pair grows wider than the gap between the columns -- - 74 against 12 at width 200 -- so with nothing drawn the stamps pair up with the wrong - neighbour and the row reads as four unrelated times. + The rule is what pairs the two stamps. UTILIZATION and MEMORY each print one, so the + row ends up holding four times, and with the rule left blank the only cue is spacing -- + which points the wrong way above 88 columns: at 200 there are 66 blanks between a + column's own two stamps but only 13 between the columns, so each column's newest time + reads as belonging to the next column's oldest. """ left, right = _stamp(first), _stamp(last) fill = max(1, width - len(left) - len(right) - 2) @@ -187,12 +153,6 @@ def _axis(width: int, first: datetime, last: datetime) -> Text: def _stamp(moment: datetime) -> str: - """`now` for the newest sample of a running job, an absolute local time otherwise. - - Collection runs every ~10s, so a running job's newest sample is always within - `pretty_date`'s `now` threshold; anything else is old enough to deserve a real time, - and a finished run then cannot be mistaken for live data. - """ if pretty_date(moment) == "now": return "now" local = moment.astimezone() @@ -200,28 +160,19 @@ def _stamp(moment: datetime) -> str: def _window(job_metrics: JobMetrics) -> Optional[tuple[datetime, datetime]]: - """Oldest and newest sample, or None if there are none.""" stamps = [t for metric in job_metrics.metrics for t in metric.timestamps] return (min(stamps), max(stamps)) if stamps else None def _samples_num(job_metrics: JobMetrics) -> int: - """How many cells the sparklines will actually occupy. - - `slices` never draws more cells than it has samples -- inventing the rest would be - fabricating data -- so a run younger than the terminal is wide fills only part of the - row. The axis has to stop where the data stops, or it claims a span nothing was - measured over and Rich widens the column to fit it. - """ + """`slices` never draws more cells than it has samples, so the axis must stop there + too -- else it claims a span nothing was measured over, and Rich widens the column.""" return max((len(metric.timestamps) for metric in job_metrics.metrics), default=0) def _metric_values(job_metrics: JobMetrics, name: str) -> List[Any]: - """Values for `name`, oldest first. - - The server returns points ordered latest to earliest. Reversing here, once, at the - boundary, is what keeps every sparkline drawn downstream running left-to-right in time. - """ + """Values for `name`, oldest first. The server sends latest-first; reversing here, once, + is what keeps every sparkline downstream running left-to-right in time.""" for metric in job_metrics.metrics: if metric.name == name: return list(reversed(metric.values)) @@ -234,11 +185,6 @@ def _latest(job_metrics: JobMetrics, name: str) -> Optional[Any]: def _gpus_num(job_metrics: JobMetrics, resources: Optional[Resources]) -> int: - """How many device rows to print. - - Prefer the offer over the metrics: it is still known when no samples have arrived, and - when the server drops GPU metrics because the device count changed mid-window. - """ if resources is not None and resources.gpus: return len(resources.gpus) detected = _latest(job_metrics, "gpus_detected_num") diff --git a/src/dstack/_internal/cli/utils/sparkline.py b/src/dstack/_internal/cli/utils/sparkline.py index 0487320761..abb7932954 100644 --- a/src/dstack/_internal/cli/utils/sparkline.py +++ b/src/dstack/_internal/cli/utils/sparkline.py @@ -1,24 +1,18 @@ -"""Sparklines for CLI tables.""" - from typing import List, Optional, Sequence from rich.console import Console from rich.text import Text SPARKS = "▁▂▃▄▅▆▇" -"""Stops at 7/8 height on purpose: a full block fills its cell to the top edge, so a column -of them in consecutive rows fuses into one mass and the rows stop reading as separate -series.""" +"""No full block: it fills the cell to the top edge, fusing consecutive rows into one mass.""" ASCII_SPARKS = "_.:-=+*#%@" NO_DATA = "no data" Ramp = Sequence[tuple[float, str]] -# Colour encodes scope. Device metrics share one ramp because GPU utilization and GPU memory -# are two views of the same question; host metrics get a cool one so a job row is never -# mistaken for a device row, and because the economics differ -- a host CPU at 10% is normal, -# not wasted money. +# Colour encodes scope: device metrics share one ramp, host metrics another, so a job row is +# never mistaken for a device row. GPU_RAMP: Ramp = ((25, "grey42"), (50, "chartreuse4"), (75, "chartreuse3"), (101, "green1")) HOST_RAMP: Ramp = ( (50, "steel_blue3"), @@ -36,7 +30,6 @@ def ramp_style(value: float, ramp: Ramp) -> str: def supports_unicode(console: Console) -> bool: - """Whether block glyphs will survive the console's encoding.""" try: SPARKS.encode(console.encoding or "utf-8") except (UnicodeEncodeError, LookupError): @@ -45,16 +38,11 @@ def supports_unicode(console: Console) -> bool: def slices(values: Sequence[float], width: int) -> List[tuple[float, float]]: - """`(peak, mean)` per cell, oldest first. - - Two numbers because a cell draws both: glyph height is the peak, colour is the mean. - Height alone cannot separate a card that *touched* 100% for a few seconds from one that - *held* it for minutes -- both draw full. + """`(peak, mean)` per cell, oldest first -- height is the peak, colour the mean, so a + card that *touched* 100% reads differently from one that *held* it. - Cells cover the whole series rather than its tail: metrics arrive about every 10s, so - taking the last `width` samples would cover a couple of minutes instead of the hour the - caller asked for. The last cell is the latest sample, since the number printed beside - the sparkline is that reading and the two must not contradict each other. + Cells span the whole series, not its tail. The last cell is the latest sample, so it + cannot contradict the number printed beside the sparkline. """ vals = list(values) if width < 1: @@ -72,9 +60,7 @@ def slices(values: Sequence[float], width: int) -> List[tuple[float, float]]: def no_data() -> Text: - """Spelled out rather than the `-` used elsewhere in the CLI: next to sparklines a bare - dash reads as a stray character, and "n/a" would be wrong -- the metric applies, it just - has not arrived yet.""" + """Spelled out: next to sparklines a bare `-` reads as a stray glyph.""" return Text(NO_DATA, style="grey58") @@ -85,8 +71,7 @@ def sparkline( vmax: float = 100.0, ascii_only: bool = False, ) -> Text: - """A sparkline on a fixed 0..vmax scale, so glyph height always means the same thing as - the number beside it and rows stay comparable with each other.""" + """Fixed 0..vmax scale, never autoscaled: height means the same thing on every row.""" if not values: return no_data() glyphs = ASCII_SPARKS if ascii_only else SPARKS diff --git a/src/tests/_internal/cli/commands/test_metrics.py b/src/tests/_internal/cli/commands/test_metrics.py index 1e3e59d20d..2309a480fa 100644 --- a/src/tests/_internal/cli/commands/test_metrics.py +++ b/src/tests/_internal/cli/commands/test_metrics.py @@ -23,7 +23,6 @@ def _run(replicas: int = 1, jobs_per_replica: int = 1): class TestJobSelection: def test_defaults_to_the_first_job_of_the_first_replica(self): - """Same shape as `dstack logs`: one job, --replica/--job to pick another.""" job = _get_job(_run(replicas=3), replica_num=0, job_num=0) assert (job.job_spec.replica_num, job.job_spec.job_num) == (0, 0) @@ -38,9 +37,6 @@ def test_unknown_job_is_an_error(self): class TestMetricsRequest: def test_limit_is_sent_explicitly(self): - """The endpoint declares `limit: int = 1`, not Optional, so omitting it caps the - response at a single sample -- which renders as one glyph however long the run has - been up, and looks plausible rather than broken.""" api = MagicMock() api.project = "main" api.client.metrics.get_job_metrics.return_value = JobMetrics(metrics=[]) diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py index e45537df78..bfd332b956 100644 --- a/src/tests/_internal/cli/utils/test_metrics.py +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -78,8 +78,6 @@ def _row(output: str, label: str) -> str: class TestMetricValues: def test_reverses_server_order_to_oldest_first(self): - """The service returns points latest to earliest; everything downstream assumes - the opposite, so this is the one place the series is flipped.""" job_metrics = JobMetrics( metrics=[ Metric( @@ -105,8 +103,6 @@ def test_each_row_carries_both_of_its_numbers(self): class TestTimeAxis: def test_ends_at_now_while_the_job_is_running(self): - """Collection is every ~10s, so a running job's newest sample is always `now` -- - and a finished run's is not, which stops a stale table reading as live.""" assert _lines(_render(_job(gpus=1), _job_metrics(gpus=1)))[-1].endswith("now") def test_older_samples_get_an_absolute_local_time(self): @@ -139,8 +135,6 @@ def test_axis_stops_where_a_young_run_stops(self): class TestNoData: def test_devices_are_listed_even_with_no_samples(self): - """The device list comes from the offer, so the shape of the run is known even - when none of its numbers are.""" output = _render(_job(gpus=4), JobMetrics(metrics=[])) assert sum(1 for ln in _lines(output) if "gpu=" in ln) == 4 assert "no data" in output diff --git a/src/tests/_internal/cli/utils/test_sparkline.py b/src/tests/_internal/cli/utils/test_sparkline.py index 6ed547d4b4..208bc4799d 100644 --- a/src/tests/_internal/cli/utils/test_sparkline.py +++ b/src/tests/_internal/cli/utils/test_sparkline.py @@ -24,8 +24,6 @@ def test_last_cell_is_the_latest_sample(self): assert slices([79.0] * 997 + [0.0, 0.0, 0.0], 10)[-1] == (0.0, 0.0) def test_height_is_the_peak_and_colour_is_the_mean(self): - """Height alone cannot separate a card that touched 100% briefly from one that - held it -- both draw full.""" peak, mean = slices([0.0] * 17 + [100.0, 0.0], 2)[0] assert peak == 100.0 assert mean < 10.0 @@ -40,8 +38,6 @@ def test_width_and_fixed_scale(self): assert sparkline([100] * 60, 10, GPU_RAMP).plain == SPARKS[-1] * 10 def test_never_fills_the_cell_to_the_top(self): - """A full block touches the cell's top edge, so a column of them in consecutive - rows fuses into one mass and eight GPU rows stop reading as separate series.""" assert "█" not in SPARKS assert "█" not in sparkline([100] * 60, 10, GPU_RAMP).plain @@ -62,7 +58,6 @@ def test_ascii_fallback(self): class TestRamps: def test_scope_is_never_ambiguous(self): - """Colour marks scope, so a job row must not share a colour with a device row.""" for value in (0, 10, 30, 60, 85, 100): assert ramp_style(value, GPU_RAMP) != ramp_style(value, HOST_RAMP) From f819ab033108f77aad2682d31c0e7ac1d5869a6e Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 5 Aug 2026 15:57:45 +0200 Subject: [PATCH 8/8] Drop non-Unicode support and fix the young-run axis `ps`, `fleet` and `project` all crash on a terminal that cannot encode their output, so a private ASCII fallback here bought nothing but an `ascii_only` parameter threaded through six functions. Removed, along with `ASCII_SPARKS` and the encoding probe. Rewrites the tests around one `make_run(shape, samples, state, gpus)` generator: 29 test functions become 14, and the sparkline module is now covered through the rendered table rather than separately. The new shapes cover what the old suite never did -- spikes, idle, colour, and terminated runs. That immediately caught a real bug. `_axis` forced at least one rule character, so when two dates did not fit the cells available it overflowed anyway, widening the column and pushing MEMORY out of line with the charts above it. It affects roughly the first three minutes of every run at any terminal width. The old tests could not see it: they stamped every sample with the same instant, so both ends rendered as the three-character `now` and always fit. The axis now drops the date, then the rule, and finally itself rather than overflow. Co-Authored-By: Claude Opus 5 (1M context) --- src/dstack/_internal/cli/utils/metrics.py | 59 ++-- src/dstack/_internal/cli/utils/sparkline.py | 16 +- src/tests/_internal/cli/utils/test_metrics.py | 280 ++++++++++-------- .../_internal/cli/utils/test_sparkline.py | 67 ----- 4 files changed, 181 insertions(+), 241 deletions(-) delete mode 100644 src/tests/_internal/cli/utils/test_sparkline.py diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py index 0cfba7532d..ccf840a99a 100644 --- a/src/dstack/_internal/cli/utils/metrics.py +++ b/src/dstack/_internal/cli/utils/metrics.py @@ -6,14 +6,7 @@ from rich.text import Text from dstack._internal.cli.utils.common import console -from dstack._internal.cli.utils.sparkline import ( - GPU_RAMP, - HOST_RAMP, - Ramp, - no_data, - sparkline, - supports_unicode, -) +from dstack._internal.cli.utils.sparkline import GPU_RAMP, HOST_RAMP, Ramp, no_data, sparkline from dstack._internal.core.models.instances import Resources from dstack._internal.core.models.metrics import JobMetrics from dstack._internal.core.models.runs import Job @@ -46,7 +39,6 @@ def _spark_width(console_width: int) -> int: def get_metrics_table( job: Job, metrics: JobMetrics, console_width: Optional[int] = None ) -> RenderableType: - ascii_only = not supports_unicode(console) resources = _get_resources(job) width = _spark_width(console_width or console.width) @@ -58,15 +50,15 @@ def get_metrics_table( table.add_row( "cpu", - _cpu_cell(metrics, resources, width, ascii_only), - _memory_cell(metrics, resources, width, ascii_only), + _cpu_cell(metrics, resources, width), + _memory_cell(metrics, resources, width), ) table.add_row("", "", "") # host and devices are different things; separate them for index in range(_gpus_num(metrics, resources)): table.add_row( f"gpu={index}", - _gpu_util_cell(metrics, index, width, ascii_only), - _gpu_memory_cell(metrics, resources, index, width, ascii_only), + _gpu_util_cell(metrics, index, width), + _gpu_memory_cell(metrics, resources, index, width), ) window = _window(metrics) if window is not None: @@ -76,9 +68,7 @@ def get_metrics_table( return table -def _cpu_cell( - job_metrics: JobMetrics, resources: Optional[Resources], width: int, ascii_only: bool -) -> Text: +def _cpu_cell(job_metrics: JobMetrics, resources: Optional[Resources], width: int) -> Text: values = _metric_values(job_metrics, "cpu_usage_percent") if not values: return no_data() @@ -88,17 +78,15 @@ def _cpu_cell( label = f"{values[-1]:.0f}%" if cpus: label += f" of {cpus}" - return _cell(sparkline(values, width, HOST_RAMP, ascii_only=ascii_only), label) + return _cell(sparkline(values, width, HOST_RAMP), label) -def _memory_cell( - job_metrics: JobMetrics, resources: Optional[Resources], width: int, ascii_only: bool -) -> Text: +def _memory_cell(job_metrics: JobMetrics, resources: Optional[Resources], width: int) -> Text: values = _metric_values(job_metrics, "memory_working_set_bytes") if not values: return no_data() total = resources.memory_mib * 1024 * 1024 if resources else None - return _level_cell(values, total, width, ascii_only, HOST_RAMP) + return _level_cell(values, total, width, HOST_RAMP) def _gpu_memory_cell( @@ -106,7 +94,6 @@ def _gpu_memory_cell( resources: Optional[Resources], index: int, width: int, - ascii_only: bool, ) -> Text: values = _metric_values(job_metrics, f"gpu_memory_usage_bytes_gpu{index}") if not values: @@ -114,24 +101,22 @@ def _gpu_memory_cell( total = None if resources and index < len(resources.gpus): total = resources.gpus[index].memory_mib * 1024 * 1024 - return _level_cell(values, total, width, ascii_only, GPU_RAMP) + return _level_cell(values, total, width, GPU_RAMP) -def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int, ascii_only: bool) -> Text: +def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int) -> Text: values = _metric_values(job_metrics, f"gpu_util_percent_gpu{index}") if not values: return no_data() - return _cell(sparkline(values, width, GPU_RAMP, ascii_only=ascii_only), f"{values[-1]:.0f}%") + return _cell(sparkline(values, width, GPU_RAMP), f"{values[-1]:.0f}%") -def _level_cell( - values: List[float], total: Optional[float], width: int, ascii_only: bool, ramp: Ramp -) -> Text: +def _level_cell(values: List[float], total: Optional[float], width: int, ramp: Ramp) -> Text: percents = [v / total * 100 for v in values] if total else values label = format_memory(values[-1], 0) if total: label += f"/{format_memory(total, 0)}" - return _cell(sparkline(percents, width, ramp, ascii_only=ascii_only), label) + return _cell(sparkline(percents, width, ramp), label) def _cell(spark: Text, label: str) -> Text: @@ -139,24 +124,32 @@ def _cell(spark: Text, label: str) -> Text: def _axis(width: int, first: datetime, last: datetime) -> Text: - """` ┄┄┄ `, as wide as the sparkline above it. + """` ┄┄┄ `, never wider than the sparkline above it. The rule is what pairs the two stamps. UTILIZATION and MEMORY each print one, so the row ends up holding four times, and with the rule left blank the only cue is spacing -- which points the wrong way above 88 columns: at 200 there are 66 blanks between a column's own two stamps but only 13 between the columns, so each column's newest time reads as belonging to the next column's oldest. + + A run draws one cell per sample, so for its first few minutes there are fewer cells + than two dates need. Dropping the date keeps the axis inside its cell; overflowing + instead widens the column and pulls MEMORY out of line with the charts. """ left, right = _stamp(first), _stamp(last) - fill = max(1, width - len(left) - len(right) - 2) + if len(left) + len(right) + 3 > width: + left, right = _stamp(first, clock_only=True), _stamp(last, clock_only=True) + if len(left) + len(right) + 2 > width: + return Text("") + fill = width - len(left) - len(right) - 2 return Text(f"{left} " + AXIS_RULE * fill + f" {right}", style="grey42") -def _stamp(moment: datetime) -> str: +def _stamp(moment: datetime, clock_only: bool = False) -> str: if pretty_date(moment) == "now": return "now" local = moment.astimezone() - return f"{local.day} {local:%b %H:%M}" + return f"{local:%H:%M}" if clock_only else f"{local.day} {local:%b %H:%M}" def _window(job_metrics: JobMetrics) -> Optional[tuple[datetime, datetime]]: diff --git a/src/dstack/_internal/cli/utils/sparkline.py b/src/dstack/_internal/cli/utils/sparkline.py index abb7932954..93beddfc5f 100644 --- a/src/dstack/_internal/cli/utils/sparkline.py +++ b/src/dstack/_internal/cli/utils/sparkline.py @@ -1,12 +1,10 @@ from typing import List, Optional, Sequence -from rich.console import Console from rich.text import Text SPARKS = "▁▂▃▄▅▆▇" """No full block: it fills the cell to the top edge, fusing consecutive rows into one mass.""" -ASCII_SPARKS = "_.:-=+*#%@" NO_DATA = "no data" Ramp = Sequence[tuple[float, str]] @@ -29,14 +27,6 @@ def ramp_style(value: float, ramp: Ramp) -> str: return ramp[-1][1] -def supports_unicode(console: Console) -> bool: - try: - SPARKS.encode(console.encoding or "utf-8") - except (UnicodeEncodeError, LookupError): - return False - return True - - def slices(values: Sequence[float], width: int) -> List[tuple[float, float]]: """`(peak, mean)` per cell, oldest first -- height is the peak, colour the mean, so a card that *touched* 100% reads differently from one that *held* it. @@ -69,15 +59,13 @@ def sparkline( width: int, ramp: Optional[Ramp] = None, vmax: float = 100.0, - ascii_only: bool = False, ) -> Text: """Fixed 0..vmax scale, never autoscaled: height means the same thing on every row.""" if not values: return no_data() - glyphs = ASCII_SPARKS if ascii_only else SPARKS text = Text() for peak, mean in slices(values, width): - index = int(max(0.0, min(vmax, peak)) / vmax * (len(glyphs) - 1)) + index = int(max(0.0, min(vmax, peak)) / vmax * (len(SPARKS) - 1)) shade = max(0.0, min(vmax, mean)) / vmax * 100 - text.append(glyphs[index], style=ramp_style(shade, ramp) if ramp else "cyan") + text.append(SPARKS[index], style=ramp_style(shade, ramp) if ramp else "cyan") return text diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py index bfd332b956..2f9d1e6b44 100644 --- a/src/tests/_internal/cli/utils/test_metrics.py +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -1,162 +1,188 @@ import re from datetime import datetime, timedelta, timezone -from typing import List +from typing import List, Tuple from unittest.mock import MagicMock import pytest from rich.console import Console from rich.theme import Theme -from dstack._internal.cli.utils.metrics import ( - AXIS_RULE, - MAX_SPARK_WIDTH, - MIN_SPARK_WIDTH, - _axis, - _metric_values, - _spark_width, - format_memory, - get_metrics_table, -) +from dstack._internal.cli.utils.metrics import format_memory, get_metrics_table from dstack._internal.cli.utils.sparkline import SPARKS from dstack._internal.core.models.metrics import JobMetrics, Metric GIB = 1024**3 - - -def _metric(name: str, values: List[float]) -> Metric: - """Values as the server returns them: latest first.""" - now = datetime.now(timezone.utc) - return Metric(name=name, timestamps=[now] * len(values), values=list(reversed(values))) - - -def _job_metrics( - gpus: int = 0, +CAPACITY_GB = 80 + +# utilization percent, and memory as a fraction of capacity, over `t` in 0..1 oldest to newest +SHAPES = { + "idle": (lambda t: 1.0, lambda t: 0.05), + "spike": (lambda t: 100.0 if 0.49 < t < 0.51 else 2.0, lambda t: 0.5), + "ramp": (lambda t: t * 100.0, lambda t: t * 0.5), + "saturated": (lambda t: 95.0, lambda t: 0.95), + "low": (lambda t: 19.0, lambda t: 0.19), +} + + +def make_run( + shape: str = "saturated", + samples: int = 360, + state: str = "running", + gpus: int = 1, cpus: int = 8, - cpu_pct: float = 50.0, - gpu_util: float = 80.0, - samples: int = 60, -) -> JobMetrics: +) -> Tuple[MagicMock, JobMetrics]: + """A job and its metrics. `state` decides whether the newest sample reads as `now`.""" + newest = datetime.now(timezone.utc) + if state == "terminated": + newest -= timedelta(hours=2) + timestamps = [newest - timedelta(seconds=10 * i) for i in range(samples)] + + def series(fn) -> List[float]: + oldest_first = [fn(i / max(1, samples - 1)) for i in range(samples)] + return list(reversed(oldest_first)) # the server returns points newest first + + util, memory = SHAPES[shape] metrics = [ - _metric("cpu_usage_percent", [cpu_pct * cpus] * samples), - _metric("memory_working_set_bytes", [4 * GIB] * samples), + Metric( + name="cpu_usage_percent", + timestamps=timestamps, + values=series(lambda t: util(t) * cpus), + ), + Metric( + name="memory_working_set_bytes", + timestamps=timestamps, + values=series(lambda t: memory(t) * 32 * GIB), + ), ] for index in range(gpus): - metrics.append(_metric(f"gpu_util_percent_gpu{index}", [gpu_util] * samples)) - metrics.append(_metric(f"gpu_memory_usage_bytes_gpu{index}", [60 * GIB] * samples)) - return JobMetrics(metrics=metrics) - + metrics.append( + Metric(name=f"gpu_util_percent_gpu{index}", timestamps=timestamps, values=series(util)) + ) + metrics.append( + Metric( + name=f"gpu_memory_usage_bytes_gpu{index}", + timestamps=timestamps, + values=series(lambda t: memory(t) * CAPACITY_GB * GIB), + ) + ) -def _job(gpus: int = 0, cpus: int = 8, with_resources: bool = True): job = MagicMock() submission = MagicMock() - if with_resources: - resources = MagicMock() - resources.cpus, resources.memory_mib = cpus, 32 * 1024 - resources.gpus = [MagicMock(memory_mib=80 * 1024) for _ in range(gpus)] - submission.job_runtime_data.offer.instance.resources = resources - else: - submission.job_runtime_data = None - submission.job_provisioning_data = None + resources = MagicMock() + resources.cpus, resources.memory_mib = cpus, 32 * 1024 + resources.gpus = [MagicMock(memory_mib=CAPACITY_GB * 1024) for _ in range(gpus)] + submission.job_runtime_data.offer.instance.resources = resources job.job_submissions = [submission] - return job + return job, JobMetrics(metrics=metrics) -def _render(job, metrics: JobMetrics, width: int = 200) -> str: - console = Console(width=width, theme=Theme({"secondary": "grey58"}), no_color=True) +def render(job, metrics: JobMetrics, width: int = 200, color: bool = False) -> str: + console = Console( + width=width, + theme=Theme({"secondary": "grey58"}), + no_color=not color, + force_terminal=color, + color_system="truecolor" if color else None, + ) with console.capture() as capture: console.print(get_metrics_table(job, metrics, console_width=width)) return capture.get() -def _lines(output: str) -> List[str]: +def lines(output: str) -> List[str]: return [line.rstrip() for line in output.splitlines() if line.strip()] -def _row(output: str, label: str) -> str: - return next(line for line in _lines(output) if line.strip().startswith(label)) - - -class TestMetricValues: - def test_reverses_server_order_to_oldest_first(self): - job_metrics = JobMetrics( - metrics=[ - Metric( - name="gpu_util_percent_gpu0", - timestamps=[datetime.now(timezone.utc)] * 3, - values=[30, 20, 10], - ) - ] - ) - assert _metric_values(job_metrics, "gpu_util_percent_gpu0") == [10, 20, 30] - - def test_missing_metric_is_empty(self): - assert _metric_values(JobMetrics(metrics=[]), "cpu_usage_percent") == [] - - -class TestLayout: - def test_each_row_carries_both_of_its_numbers(self): - output = _render(_job(gpus=1), _job_metrics(gpus=1, cpu_pct=25.0, gpu_util=77.0)) - assert "25% of 8" in _row(output, "cpu") - assert "77%" in _row(output, "gpu=0") - assert "60GB/80GB" in _row(output, "gpu=0") - - -class TestTimeAxis: - def test_ends_at_now_while_the_job_is_running(self): - assert _lines(_render(_job(gpus=1), _job_metrics(gpus=1)))[-1].endswith("now") - - def test_older_samples_get_an_absolute_local_time(self): - now = datetime.now(timezone.utc) - assert _axis(40, now - timedelta(hours=2), now).plain.endswith("now") - stopped = _axis(40, now - timedelta(hours=2), now - timedelta(hours=1)).plain - assert "now" not in stopped - assert ":" in stopped # a real clock time, not an age - - def test_axis_aligns_with_the_sparklines(self): - output = _render(_job(gpus=1), _job_metrics(gpus=1)) - axis, cpu_row = _lines(output)[-1], _row(output, "cpu") - assert axis.index(axis.strip()[0]) == min(cpu_row.index(g) for g in SPARKS if g in cpu_row) - - def test_no_axis_when_there_are_no_samples(self): - assert AXIS_RULE not in _render(_job(gpus=1), JobMetrics(metrics=[])) - - def test_axis_stops_where_a_young_run_stops(self): - """The sparkline draws one cell per sample and will not invent more, so a run - younger than the terminal is wide fills only part of the row. An axis drawn to the - requested width would claim a span nothing was measured over -- and Rich would - widen the column to fit it, pushing MEMORY sideways.""" - output = _render(_job(gpus=1), _job_metrics(gpus=1, samples=12), width=200) - axis, cpu_row = _lines(output)[-1], _row(output, "cpu") - drawn = sum(1 for char in cpu_row if char in SPARKS) // 2 # two columns per row - assert drawn == 12 # one cell per sample, not the 80 cells 200 columns would allow - # the two columns print the same axis, separated by the table's own padding +def row(output: str, label: str) -> str: + return next(line for line in lines(output) if line.strip().startswith(label)) + + +def bars(line: str) -> List[int]: + """Glyph heights of the first sparkline in `line`, left to right.""" + return [SPARKS.index(glyph) for glyph in re.findall(rf"[{SPARKS}]+", line)[0]] + + +def colours(output: str, label: str) -> set: + """Distinct colours among the glyphs of `label`'s row.""" + line = next(ln for ln in output.splitlines() if label in ln) + return {code for code, _ in re.findall(rf"\x1b\[([0-9;]+)m([{SPARKS}])", line)} + + +class TestRendering: + def test_idle_draws_flat_and_low_in_one_colour(self): + """An idle GPU is a flat low line in a single colour, not a rainbow of bands.""" + job, metrics = make_run("idle") + assert set(bars(row(render(job, metrics), "gpu=0"))) == {0} + assert len(colours(render(job, metrics, color=True), "gpu=0")) == 1 + + def test_a_spike_survives_bucketing(self): + """One sample at 100% among 360 still draws tall; averaging would erase it.""" + job, metrics = make_run("spike") + assert max(bars(row(render(job, metrics), "gpu=0"))) == len(SPARKS) - 1 + + def test_a_ramp_climbs_left_to_right(self): + """The server sends points newest first, so a missing reversal mirrors every chart + and nothing else on screen would give it away.""" + job, metrics = make_run("ramp") + heights = bars(row(render(job, metrics), "gpu=0")) + assert heights == sorted(heights) + assert heights[0] < heights[-1] + + def test_height_is_a_fraction_of_capacity_not_of_the_window(self): + """19% of capacity looks nearly empty. Rescaling to the window's own maximum would + draw a steady 154GB of 800GB as a full bar.""" + job, metrics = make_run("low") + assert max(bars(row(render(job, metrics), "gpu=0"))) <= 1 + + def test_the_number_matches_the_last_bar(self): + """The printed value is the newest sample and the right-hand bar draws that same + sample, so a value that just dropped cannot show tall beside a 0.""" + job, metrics = make_run("ramp") + gpu = row(render(job, metrics), "gpu=0") + assert "100%" in gpu + assert bars(gpu)[-1] == len(SPARKS) - 1 + + def test_no_data_is_not_zero(self): + """A device we have no metrics for reads differently from an idle one, and its row + is listed either way -- the device list comes from the offer.""" + job, _ = make_run("idle", gpus=2) + missing = render(job, JobMetrics(metrics=[])) + assert "no data" in missing + assert not re.findall(rf"[{SPARKS}]", missing) + assert sum(1 for line in lines(missing) if "gpu=" in line) == 2 + assert "1%" in render(*make_run("idle", gpus=2)) + + +class TestWindow: + @pytest.mark.parametrize("samples", [12, 360], ids=["two-minutes", "an-hour"]) + @pytest.mark.parametrize("state", ["running", "terminated"]) + def test_draws_only_what_was_measured(self, samples: int, state: str): + """A young run fills part of the row and the timeline stops with it. Drawn to the + full width it would claim a span nothing was measured over, and Rich would widen + the column to fit, pulling MEMORY out of line.""" + job, metrics = make_run("ramp", samples=samples, state=state) + output = render(job, metrics, width=200) + drawn = len(bars(row(output, "cpu"))) + assert drawn == min(samples, 80) # 80 is MAX_SPARK_WIDTH + axis = lines(output)[-1] assert [len(segment) for segment in re.split(r"\s{3,}", axis.strip())] == [drawn, drawn] - -class TestNoData: - def test_devices_are_listed_even_with_no_samples(self): - output = _render(_job(gpus=4), JobMetrics(metrics=[])) - assert sum(1 for ln in _lines(output) if "gpu=" in ln) == 4 - assert "no data" in output - - def test_measured_zero_is_distinguishable_from_missing(self): - zeros = _render(_job(gpus=1), _job_metrics(gpus=1, gpu_util=0.0, cpu_pct=0.0)) - assert "0%" in zeros - assert zeros != _render(_job(gpus=1), JobMetrics(metrics=[])) - - def test_no_resources_and_no_metrics_renders_no_device_rows(self): - assert "gpu=" not in _render(_job(with_resources=False), JobMetrics(metrics=[])) - - -class TestSparkWidth: - def test_clamped_at_both_ends(self): - assert _spark_width(20) == MIN_SPARK_WIDTH - assert _spark_width(10_000) == MAX_SPARK_WIDTH - - def test_table_fits_the_terminal(self): - for width in (80, 100, 140, 190, 240): - output = _render(_job(gpus=8), _job_metrics(gpus=8), width=width) - assert max(len(ln.rstrip()) for ln in output.splitlines()) <= width + @pytest.mark.parametrize("state,live", [("running", True), ("terminated", False)]) + def test_a_finished_run_cannot_look_live(self, state: str, live: bool): + job, metrics = make_run("saturated", state=state) + axis = lines(render(job, metrics))[-1] + assert axis.endswith("now") == live + if not live: + assert ":" in axis # a real clock time, not an age + + +@pytest.mark.parametrize("width", [80, 100, 140, 190, 240]) +def test_fits_every_terminal_width(width: int): + """Nothing wraps or gets truncated, on the widest realistic row: eight GPUs.""" + job, metrics = make_run("saturated", gpus=8) + output = render(job, metrics, width=width) + assert max(len(line.rstrip()) for line in output.splitlines()) <= width + assert "…" not in output @pytest.mark.parametrize( diff --git a/src/tests/_internal/cli/utils/test_sparkline.py b/src/tests/_internal/cli/utils/test_sparkline.py deleted file mode 100644 index 208bc4799d..0000000000 --- a/src/tests/_internal/cli/utils/test_sparkline.py +++ /dev/null @@ -1,67 +0,0 @@ -from dstack._internal.cli.utils.sparkline import ( - GPU_RAMP, - HOST_RAMP, - NO_DATA, - SPARKS, - ramp_style, - slices, - sparkline, -) - - -class TestSlices: - def test_cells_cover_the_whole_series_not_its_tail(self): - """Samples arrive ~10s apart, so taking the last `width` of them would cover a - couple of minutes instead of the hour the caller asked for.""" - cells = slices(list(range(1000)), 10) - assert len(cells) == 10 - assert cells[0][0] < 200 # first cell is early history, not a recent sample - - def test_last_cell_is_the_latest_sample(self): - """The number printed beside the sparkline is that reading; if the last cell were - its slice's peak instead, a value that just dropped would draw tall next to a - label reading 0.""" - assert slices([79.0] * 997 + [0.0, 0.0, 0.0], 10)[-1] == (0.0, 0.0) - - def test_height_is_the_peak_and_colour_is_the_mean(self): - peak, mean = slices([0.0] * 17 + [100.0, 0.0], 2)[0] - assert peak == 100.0 - assert mean < 10.0 - - def test_shorter_than_width_is_returned_as_is(self): - assert slices([1, 2, 3], 10) == [(1, 1), (2, 2), (3, 3)] - - -class TestSparkline: - def test_width_and_fixed_scale(self): - assert sparkline([0] * 60, 10, GPU_RAMP).plain == SPARKS[0] * 10 - assert sparkline([100] * 60, 10, GPU_RAMP).plain == SPARKS[-1] * 10 - - def test_never_fills_the_cell_to_the_top(self): - assert "█" not in SPARKS - assert "█" not in sparkline([100] * 60, 10, GPU_RAMP).plain - - def test_height_means_fraction_of_capacity_not_of_the_window(self): - """Fitting the axis to the window would draw 154GB of 800GB as a full bar.""" - climbing = [(10 + i * 0.145) / 800 * 100 for i in range(1000)] - assert sparkline(climbing, 10, GPU_RAMP).plain[-1] in SPARKS[:3] - - def test_measured_zero_is_not_missing_data(self): - assert sparkline([0] * 60, 10, GPU_RAMP).plain == SPARKS[0] * 10 - assert sparkline([], 10, GPU_RAMP).plain == NO_DATA - - def test_ascii_fallback(self): - text = sparkline([50] * 60, 10, GPU_RAMP, ascii_only=True).plain - assert len(text) == 10 - assert " " not in text # a blank would read as missing, not as a measured zero - - -class TestRamps: - def test_scope_is_never_ambiguous(self): - for value in (0, 10, 30, 60, 85, 100): - assert ramp_style(value, GPU_RAMP) != ramp_style(value, HOST_RAMP) - - def test_the_low_end_is_a_single_colour(self): - """Each glyph is coloured by its own value, so splitting "low" across bands makes - an idle GPU draw as several colours at once.""" - assert len({ramp_style(v, GPU_RAMP) for v in (0, 1, 6, 12, 24)}) == 1