From 5a01732c71b059b5bba89ba24e16032718396428 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:29:32 +0000 Subject: [PATCH 01/12] tools/eval: add efficiency evaluation (bare vs yeto, verl and Miles) Eight arms per model preset ({bare,yeto} x {verl,miles} x {sync,async}) generated from one shared spec with a control fingerprint. Bare arms run the frameworks' own entry points (verl main_ppo / fully_async_main, Miles train.py / train_async.py) in the same images yeto uses; yeto arms run verl island_entry and the yeto launcher. Parsers normalize verl FileLogger, Miles perf lines and yeto tapes into per-round records (missing fields are marked, never estimated); the report gives yeto-vs-bare deltas with min-max range verdicts. Presets: Qwen3-0.6B (E0.1) and Qwen3-4B-Instruct-2507 (E0.2). CLI: eff-plan, eff-run, eff-parse, eff-report. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 338 +++++++++++++++++++++ tools/eval/__main__.py | 93 ++++++ tools/eval/efficiency/__init__.py | 24 ++ tools/eval/efficiency/eff_prep.py | 65 ++++ tools/eval/efficiency/eff_reward.py | 39 +++ tools/eval/efficiency/metrics.py | 136 +++++++++ tools/eval/efficiency/modal_eff.py | 179 +++++++++++ tools/eval/efficiency/parse.py | 420 ++++++++++++++++++++++++++ tools/eval/efficiency/prices.json | 6 + tools/eval/efficiency/report.py | 205 +++++++++++++ tools/eval/efficiency/spec.py | 451 ++++++++++++++++++++++++++++ 11 files changed, 1956 insertions(+) create mode 100644 tests/test_tools_eval_efficiency.py create mode 100644 tools/eval/efficiency/__init__.py create mode 100644 tools/eval/efficiency/eff_prep.py create mode 100644 tools/eval/efficiency/eff_reward.py create mode 100644 tools/eval/efficiency/metrics.py create mode 100644 tools/eval/efficiency/modal_eff.py create mode 100644 tools/eval/efficiency/parse.py create mode 100644 tools/eval/efficiency/prices.json create mode 100644 tools/eval/efficiency/report.py create mode 100644 tools/eval/efficiency/spec.py diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py new file mode 100644 index 00000000..a7ff78f6 --- /dev/null +++ b/tests/test_tools_eval_efficiency.py @@ -0,0 +1,338 @@ +"""CPU tests of the efficiency evaluation (tools/eval/efficiency, S26 E0.1). + +The synthetic logs below follow the formats the parsers cite: + +* verl FileLogger rows ``{"step": N, "data": {...}}`` -- fork acad9875 + ``verl/utils/tracking.py`` ``FileLogger.log``; console ``step:N - k:v`` -- + ``verl/utils/logger/aggregate_logger.py`` ``concat_dict_to_str``; keys as in the + real T-EVAL file ``SESSION25/logs/T-EVAL/eval-e1-verl-c/.../verl-file-logger-0.jsonl`` + and ``metric_utils.compute_timing_metrics`` (``timing_s/*``); +* Miles ``perf N: {dict}`` lines -- copied in shape from the real T-EVAL log + ``SESSION25/logs/T-EVAL/eval-e1-miles-a/raw/launch.ts.log`` (rollout side + ``metrics.py:93``, train side ``train_metric_utils.py:56``); +* yeto tape events -- field names as in the T-EVAL tapes (``rl_round_wall``, + ``rl_local_round``, ``rl_round_trained``, ``rl_timeline_span``); +* NVML CSV -- ``nvidia-smi --query-gpu=... --format=csv`` output and the + ``miles_local.sh`` ``,`` prefixed form. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tools.eval import __main__ as cli +from tools.eval.efficiency import eff_reward, metrics, parse, report, spec + +SPEC = spec.get_spec("qwen3-0.6b") + + +# ====================================================================== spec +def test_arms_cover_both_systems_frameworks_and_modes(): + assert len(spec.ARMS) == 8 + for name in spec.ARMS: + arm = spec.Arm.parse(name) + assert spec.Arm.parse(arm.peer).system != arm.system + with pytest.raises(ValueError): + spec.Arm.parse("bare-verl-fast") + + +def test_every_arm_carries_the_shared_values(): + plans = {a: spec.plan(SPEC, a, "r1") for a in spec.ARMS} + assert len({p["fingerprint"] for p in plans.values()}) == 1 + for name, p in plans.items(): + text = p.get("script") or " ".join(p.get("argv") or []) or p["launcher_args"] + assert p["cards"]["total"] == (1 if p["mode"] == "sync" else 2), name + if p["system"] == "bare": + assert "Qwen3-0.6B" in text and "/root/eff-data/train" in text, name + else: + assert SPEC.model_revision in text and SPEC.data_revision in text, name + for value in (SPEC.rounds, SPEC.groups_per_round, SPEC.samples_per_group, SPEC.max_response_len, + SPEC.lr, SPEC.lora_r, SPEC.seed): + assert str(value) in text, (name, value) + + +def test_fingerprint_changes_with_a_shared_value_only(): + base = spec.control_fingerprint(SPEC) + assert spec.control_fingerprint(spec.get_spec("qwen3-0.6b", repeats=5)) == base + assert spec.control_fingerprint(spec.get_spec("qwen3-0.6b", lr=2e-5)) != base + + +def test_bare_entry_points_follow_the_official_recipes(): + sync = spec.bare_script(SPEC, spec.Arm.parse("bare-verl-sync"), "r1") + async_ = spec.bare_script(SPEC, spec.Arm.parse("bare-verl-async"), "r1") + assert "-m verl.trainer.main_ppo" in sync and "fully_async_main" in async_ + assert "async_training.staleness_threshold=1" in async_ and "rollout.n_gpus_per_node=1" in async_ + assert "trainer.logger=[console,file]" in sync and "VERL_FILE_LOGGER_PATH" in sync + miles_sync = spec.bare_script(SPEC, spec.Arm.parse("bare-miles-sync"), "r1") + miles_async = spec.bare_script(SPEC, spec.Arm.parse("bare-miles-async"), "r1") + assert "/root/miles/train.py $MODEL_ARGS" in miles_sync and "--colocate" in miles_sync + assert "/root/miles/train_async.py" in miles_async and "--fully-async --max-weight-staleness 1" in miles_async + assert "scripts/models/qwen3-0.6B.py" in miles_sync # the image's own model flags + for script in (sync, async_, miles_sync, miles_async): + assert "yeto" not in script.replace("yeto-effeval", "").replace("yeto/", "") + + +def test_yeto_arms_use_the_same_policy_age_and_partition(): + assert "--rl-max-policy-age 1" in " ".join(spec.yeto_verl_argv(SPEC, "async", "/t")) + assert "--rl-max-policy-age" not in " ".join(spec.yeto_verl_argv(SPEC, "sync", "/t")) + args = spec.yeto_miles_launcher_args(SPEC, "async", "r1") + assert "--gpu modal:2xa10g" in args and "--rl-placement fixed-partition --rl-rollout-gpus 1" in args + assert "--gpu modal:1xa10g" in spec.yeto_miles_launcher_args(SPEC, "sync", "r1") + + +# overrides yeto's verl translation sets and the bare recipe leaves at the default (or the reverse); +# a new key here means the two arms drifted apart: decide, then extend the list +KNOWN_VERL_DIFF = { + "actor_rollout_ref.model.lora.merge", "actor_rollout_ref.rollout.enforce_eager", + "actor_rollout_ref.rollout.full_determinism", "actor_rollout_ref.rollout.load_format", + "ray_kwargs.ray_init.num_cpus", "trainer.project_name", "trainer.use_v1", "trainer.v1.trainer_mode", + "actor_rollout_ref.rollout.mode", "async_training.staleness_threshold", "data.return_raw_chat", + "rollout.total_rollout_steps", +} + + +@pytest.mark.parametrize("mode", spec.MODES) +def test_verl_knob_diff_is_the_known_set(mode): + diff = spec.verl_knob_diff(SPEC, mode) + assert set(diff) <= KNOWN_VERL_DIFF, set(diff) - KNOWN_VERL_DIFF + + +def test_4b_preset_is_ready_for_e02(): + s4 = spec.get_spec("qwen3-4b-instruct-2507") + assert s4.gpu == "H100!" and s4.repeats >= 2 and len(s4.model_revision) == 40 + assert "--gpu modal:2xh100" in spec.yeto_miles_launcher_args(s4, "async", "r") + assert "qwen3-4B-Instruct-2507.py" in spec.bare_script(s4, spec.Arm.parse("bare-miles-sync"), "r") + + +# ====================================================================== reward +@pytest.mark.parametrize("response,label", [ + ("so \\boxed{42}", "#### 42"), ("the answer is 1,234", "x #### 1234"), ("\\boxed{3.5} then 7", "#### 3.5"), + ("no number", "#### 1"), ("\\boxed{-2}", "#### 2"), ("", None), +]) +def test_bare_reward_equals_yeto_reward(response, label): + from yeto.rl.rewards.builtin import gsm8k_grade + + assert eff_reward.grade(response, label) == gsm8k_grade(response, label) + assert eff_reward.compute_score("d", response, label) == gsm8k_grade(response, label) + + +# ====================================================================== parse: verl +def _verl_rows(n: int, *, async_: bool = False) -> str: + rows = [] + for step in range(1, n + 1): + d = {"timing_s/gen": 10.0 + step, "timing_s/old_log_prob": 1.0, "timing_s/update_actor": 3.0, + "timing_s/step": 16.0 + step, "response_length/mean": 100.0, "perf/total_num_tokens": 9000, + "critic/score/mean": 0.1 * step, "rollout_corr/rollout_is_ratio_fraction_high": 0.01} + if async_: + d.pop("timing_s/gen") + d.update({"timing_s/param_sync": 0.5, "timing_s/wait_for_enough_samples": 2.0, + "fully_async/count/dropped_stale_samples": step, "fully_async/count/total_generated_samples": + 32 * step}) + else: + d["timing_s/update_weights"] = 0.7 + rows.append(json.dumps({"step": step, "data": d})) + return "\n".join(rows) + "\n" + + +def _verl_console(n: int, t0: float = 1000.0, dt: float = 20.0) -> str: + return "".join(f"{t0 + dt * s:.3f} (TaskRunner pid=1) step:{s} - timing_s/gen:{10 + s} - critic/score/mean:0.1\n" + for s in range(1, n + 1)) + + +def test_parse_verl_sync_rounds_and_sources(): + p = parse.parse_verl(_verl_rows(3), _verl_console(3), samples_per_round=32, trainer_cards=1, mode="sync") + r = p.rounds + assert [x["round"] for x in r] == [1, 2, 3] + assert r[1]["round_s"] == 20.0 and r[0]["round_s"] is None + assert r[0]["gen_tokens"] == 3200 and r[0]["train_s"] == 4.0 and r[0]["weight_sync_s"] == 0.7 + assert p.sources["gen_tokens"].startswith("derived:") and p.sources["wait_s"] == parse.NA + assert p.sources["sampler_update_s"] == parse.NA and p.sources["version_lag"] == parse.NA + assert p.segments_raw[0]["timing_s/step"] == 17.0 + + +def test_parse_verl_async_reads_fully_async_counters(): + p = parse.parse_verl(_verl_rows(2, async_=True), "", samples_per_round=32, trainer_cards=1, mode="async") + assert p.rounds[0]["wait_s"] == 2.0 and p.rounds[0]["weight_sync_s"] == 0.5 + assert p.rounds[1]["dropped_samples"] == 2 and p.sources["gen_s"] == parse.MISSING + assert p.sources["t_end_unix"] == parse.MISSING and p.sources["version_lag"] == parse.MISSING + + +# ====================================================================== parse: Miles +def _miles_log(n: int, *, async_: bool = False) -> str: + out = [] + for rid in range(n): + ro = {"rollout/raw_reward": 0.4 + 0.01 * rid, "perf/rollout_time": 12.0, + "perf/tokens_per_gpu_per_sec": 500.0, "rollout/num_training_samples": 32} + tr = {"perf/train_wait_time": 12.5, "perf/train_time": 20.0, "perf/actor_train_time": 15.0, + "perf/actor_train_tok_per_s": 800.0, "perf/update_weights_time": 0.15, "perf/sleep_time": 1.7} + out.append(f"2026-10-20T06:36:{10 + rid:02d}Z [p-l0-modal] \x1b[36m(RolloutExecutor pid=2765)\x1b[0m " + f"[2026-10-20 06:36:{10 + rid:02d}.055 rollout_executor] metrics.py:93 - perf {rid}: {ro!r}") + out.append(f"2026-10-20T06:37:{rid * 30 % 60:02d}Z [p-l0-modal] (MegatronTrainRayActor pid=3190) " + f"[2026-10-20 06:{37 + rid // 2}:{rid * 30 % 60:02d}.147 actor_cell00000_rank00000] " + f"train_metric_utils.py:56 - perf {rid}: {tr!r}") + return "\n".join(out) + "\nunrelated line\n" + + +def test_parse_miles_merges_rollout_and_train_lines(): + p = parse.parse_miles(_miles_log(3), rollout_cards=1, mode="sync") + r = p.rounds + assert [x["round"] for x in r] == [0, 1, 2] + assert r[0]["gen_s"] == 12.0 and r[0]["train_s"] == 20.0 and r[0]["weight_sync_s"] == 0.15 + assert r[0]["gen_tokens"] == 6000.0 and r[0]["train_tokens"] == 12000.0 and r[0]["samples"] == 32 + assert r[1]["round_s"] == 30.0 and r[0]["reward_mean"] == 0.4 + assert p.sources["wait_s"] == parse.NA and p.sources["is_clip_frac"] == parse.MISSING + + +def test_parse_miles_async_keeps_wait(): + p = parse.parse_miles(_miles_log(2), rollout_cards=1, mode="async") + assert p.rounds[0]["wait_s"] == 12.5 and p.sources["dropped_samples"] == parse.MISSING + + +# ====================================================================== parse: yeto tape +def _tape(n: int, t0: float = 2000.0) -> list[dict]: + ev = [] + for rid in range(n): + end = t0 + 25.0 * (rid + 1) + ev += [{"event": "rl_timeline_span", "rollout_id": rid, "task": "generate", "kind": "compute", + "start": 100.0 * rid, "end": 100.0 * rid + 11.0}, + {"event": "rl_timeline_span", "rollout_id": rid, "task": "train", "kind": "compute", + "start": 100.0 * rid + 11.0, "end": 100.0 * rid + 21.0}, + {"event": "rl_timeline_span", "rollout_id": rid, "task": "publish", "kind": "transfer", + "start": 100.0 * rid + 21.0, "end": 100.0 * rid + 22.0}, + {"event": "rl_timeline_span", "rollout_id": rid, "task": "offload", "kind": "transfer", + "start": 100.0 * rid + 22.0, "end": 100.0 * rid + 24.0}, + {"event": "rl_round_wall", "rollout_id": rid, "round_seconds": 25.0, "time_unix": end}, + {"event": "rl_local_round", "action_tokens": 3000 + rid, "reward_mean": 0.5, "time_unix": end - 1}, + {"event": "rl_round_trained", "rollout_id": rid, "trained_samples": 32, + "mismatch": {"tis_clipfrac": 0.0}, "time_unix": end}] + return ev + + +def test_parse_yeto_tape_with_framework_tokens(): + fw = parse.parse_miles(_miles_log(3), rollout_cards=1, mode="sync") + p = parse.parse_yeto(_tape(3), mode="sync", framework_parsed=fw) + r = p.rounds + assert r[0]["gen_s"] == 11.0 and r[0]["train_s"] == 10.0 and r[0]["weight_sync_s"] == 1.0 + assert r[1]["round_s"] == 25.0 and r[2]["gen_tokens"] == 3002 and r[0]["train_tokens"] == 12000.0 + assert p.sources["train_tokens"].startswith("framework log:") and p.segments_raw[0]["offload"] == 2.0 + + +def test_parse_yeto_driver_phase_fallback(): + ev = [{"event": "rl_driver_phase", "phase": "generate", "rollout_id": 0, "time_unix": 10.0}, + {"event": "rl_driver_phase", "phase": "train", "time_unix": 18.0}, + {"event": "rl_driver_phase", "phase": "publish", "time_unix": 21.0}, + {"event": "rl_driver_phase", "phase": "generate", "rollout_id": 1, "time_unix": 22.0}, + {"event": "rl_round_wall", "rollout_id": 0, "time_unix": 22.0}] + p = parse.parse_yeto(ev, mode="sync") + assert p.rounds[0]["gen_s"] == 8.0 and p.rounds[0]["train_s"] == 3.0 and p.rounds[0]["weight_sync_s"] == 1.0 + assert "rl_driver_phase" in p.sources["gen_s"] + + +def test_parse_nvml_both_formats(): + own = ("timestamp, index, name, utilization.gpu [%], memory.used [MiB], memory.total [MiB], power.draw [W]\n" + "2026/10/20 06:36:11.055, 0, NVIDIA A10, 45 %, 6077 MiB, 23028 MiB, 81.7 W\n" + "2026/10/20 06:36:12.055, 1, NVIDIA A10, 0 %, 100 MiB, 23028 MiB, 30.1 W\n") + polled = "ta-01ABC,2026/10/20 06:36:11.055, NVIDIA A10, 44 %, 6077 MiB, 81.76 W\n" + a, b = parse.parse_nvml(own), parse.parse_nvml(polled) + assert [x["util"] for x in a] == [45.0, 0.0] and a[1]["key"] == ":1" and a[0]["mem_total_mb"] == 23028.0 + assert b[0]["key"] == "ta-01ABC:0" and b[0]["mem_mb"] == 6077.0 and b[0]["power_w"] == 81.76 + + +# ====================================================================== run dirs, metrics, report +def _nvml_csv(t0: float, n: int, util: int) -> str: + import datetime as dt + + lines = ["timestamp, index, name, utilization.gpu [%], memory.used [MiB], memory.total [MiB], power.draw [W]"] + for i in range(n): + stamp = dt.datetime.fromtimestamp(t0 + i, dt.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f")[:-3] + lines.append(f"{stamp}, 0, NVIDIA A10, {util if i % 4 else 0} %, 7000 MiB, 23028 MiB, 80 W") + return "\n".join(lines) + "\n" + + +def _run_dir(tmp: Path, arm: str, run_id: str, dt_round: float) -> Path: + plan = spec.plan(SPEC, arm, run_id) + d = tmp / run_id + spec.write_plan(d / "arm.json", plan) + raw = d / "raw" / run_id / "arm" + raw.mkdir(parents=True) + n, t0 = SPEC.rounds, 1000.0 + (raw / "meta.json").write_text(json.dumps({"rc": 0, "start_unix": t0 - 200, "end_unix": t0 + dt_round * n + 30})) + (raw / "nvml.csv").write_text(_nvml_csv(t0, int(dt_round * n) + 5, 60)) + if plan["system"] == "bare": + (raw / "main.log").write_text(_verl_console(n, t0, dt_round)) + (raw / "eff-out").mkdir() + (raw / "eff-out" / "verl-metrics.jsonl").write_text(_verl_rows(n)) + else: + out = raw / "yeto-output" + out.mkdir() + (out / "verl-file-logger-0.jsonl").write_text("".join( # steps 1..n = rollout ids 0..n-1 + json.dumps({"step": k, "data": {"global_seqlen/mean": 9000}}) + "\n" for k in range(1, n + 1))) + (out / "rl-island-0.jsonl").write_text("\n".join(json.dumps(e) for e in _tape(n, t0 - dt_round) + ) + "\n") + return d + + +def test_parse_run_metrics_and_report(tmp_path, monkeypatch): + dirs = [_run_dir(tmp_path, "bare-verl-sync", "b1", 20.0), _run_dir(tmp_path, "bare-verl-sync", "b2", 21.0), + _run_dir(tmp_path, "yeto-verl-sync", "y1", 25.0), _run_dir(tmp_path, "yeto-verl-sync", "y2", 25.0)] + for d in dirs: + assert cli.main(["eff-parse", str(d)]) == 0 + assert (d / "parsed.json").is_file() and (d / "summary.json").is_file() + s = json.loads((dirs[0] / "summary.json").read_text()) + assert s["rounds"] == SPEC.rounds and s["steady_wall_s"] == 20.0 * (SPEC.rounds - 1) + assert s["gen_tok_per_card_s"] == round(3200 / 20.0, 3) and s["bubble_frac"] == pytest.approx(0.25, abs=0.02) + assert s["usd_per_1k_samples_steady"] == round(2.0 * 20.0 * 7 / 3600 / (32 * 7) * 1000, 4) + assert s["gpu_mem_peak_mb"] == 7000.0 + assert s["missing"] == [] and "wait_s_median" in s["not_applicable"] and "drop_frac" in s["not_applicable"] + y = json.loads((dirs[2] / "summary.json").read_text()) + assert y["missing"] == [] and y["train_s_median"] == 10.0 and y["unaccounted_s_median"] == 3.0 + assert y["train_tok_per_card_s"] == round(9000 / 25.0, 3) + out = tmp_path / "report.md" + assert cli.main(["eff-report", *map(str, dirs), "--out", str(out)]) == 0 + text = out.read_text() + assert "### verl sync (bare n=2, yeto n=2)" in text + section = text.split("### verl sync")[1] + row = next(line for line in section.splitlines() if line.startswith("| round time (s)")) + assert "yeto worse" in row and "+4.5" in row + assert "| update sampler weights (s/round) | n/a | n/a | n/a | not applicable |" in text + assert "wait (s/round) | n/a" in text + + +def test_report_needs_two_repeats_and_one_spec(tmp_path): + one = [_run_dir(tmp_path, "bare-verl-sync", "b1", 20.0), _run_dir(tmp_path, "yeto-verl-sync", "y1", 25.0)] + for d in one: + cli.main(["eff-parse", str(d)]) + text = report.markdown(report.load(one), title="t") + assert "insufficient repeats (bare 1, yeto 1; need 2)" in text + runs = report.load(one) + runs[1]["fingerprint"] = "other" + with pytest.raises(ValueError, match="different specs"): + report.markdown(runs, title="t") + + +def test_verdict_rules(): + assert report.verdict([10, 11], [12, 13], True) == "yeto worse" + assert report.verdict([10, 11], [8, 9], True) == "yeto better" + assert report.verdict([10, 12], [11, 13], True) == "within run-to-run range" + assert report.verdict([10, 11], [8, 9], False) == "yeto worse" + assert report.verdict([10, None], [8, 9], True) == "missing value" + + +def test_price_comes_from_the_config(tmp_path): + path = tmp_path / "p.json" + path.write_text(json.dumps({"A10G": {"usd_per_card_hour": 3.5}})) + assert metrics.price("A10G", path) == 3.5 + with pytest.raises(KeyError): + metrics.price("H100!", path) + + +def test_cli_plan_and_unregistered_run(tmp_path, monkeypatch, capsys): + assert cli.main(["eff-plan", "qwen3-0.6b", "--arm", "bare-verl-sync"]) == 0 + assert json.loads(capsys.readouterr().out)["arms"]["bare-verl-sync"]["cards"]["total"] == 1 + monkeypatch.setattr(cli, "LEDGER", tmp_path / "ledger.md") + (tmp_path / "ledger.md").write_text("nothing registered\n") + with pytest.raises(SystemExit, match="not pre-registered"): + cli.main(["eff-run", "qwen3-0.6b", "--arm", "bare-verl-sync", "--run-id", "eff-x", "--out", str(tmp_path)]) diff --git a/tools/eval/__main__.py b/tools/eval/__main__.py index 1d047988..32ed6571 100644 --- a/tools/eval/__main__.py +++ b/tools/eval/__main__.py @@ -5,6 +5,13 @@ python -m tools.eval judge DIR/eval-e3-verl-a python -m tools.eval report DIR/* --out report.md [--html report.html] +Efficiency evaluation (S26 E0.1, package :mod:`.efficiency`): + + python -m tools.eval eff-plan qwen3-0.6b [--arm ARM] + python -m tools.eval eff-run qwen3-0.6b --arm bare-verl-sync --run-id eff-... [--out DIR] [--dry-run] + python -m tools.eval eff-parse DIR/eff-... + python -m tools.eval eff-report DIR/eff-* --out report.md + ``run`` refuses a run id that is not pre-registered in the GPU ledger (the id must appear in ``gpu-spend.md``), checks the local thread count and Ray, writes ``cell.json``, runs the cell's harness (raw data into ``raw/``), then judges. @@ -123,6 +130,71 @@ def cmd_report(args) -> int: return 0 +EFF_OUT = Path("/home/michael/work/infra-drafts/SESSION26/logs/E0") + + +def cmd_eff_plan(args) -> int: + from .efficiency import spec + + sp = spec.get_spec(args.preset) + arms = [args.arm] if args.arm else list(spec.ARMS) + doc = {"spec": sp.to_dict(), "fingerprint": spec.control_fingerprint(sp), + "arms": {a: spec.plan(sp, a, "") for a in arms}, + "verl_knob_diff": {m: spec.verl_knob_diff(sp, m) for m in spec.MODES}} + print(json.dumps(doc, indent=1, default=str)) + return 0 + + +def cmd_eff_run(args) -> int: + from .efficiency import modal_eff, spec + + sp = spec.get_spec(args.preset) + plan = spec.plan(sp, args.arm, args.run_id) + run_dir = Path(args.out) / args.run_id + if not args.dry_run: + if (run_dir / "arm.json").exists(): + raise SystemExit(f"{run_dir} already used") + _preflight(args.run_id) + plan["code_sha"] = subprocess.run(["git", "-C", str(Path(__file__).resolve().parents[2]), "rev-parse", "HEAD"], + capture_output=True, text=True).stdout.strip() + plan["images"] = spec.images() + spec.write_plan(run_dir / "arm.json", plan) + if plan["harness"] == "miles_local": + summary = modal_eff.run_miles_launcher(plan, run_dir, dry_run=args.dry_run) + if args.dry_run: + print((run_dir / "dry-run.log").read_text()) + return summary["rc"] + elif args.dry_run: + print(plan.get("script") or json.dumps(plan.get("argv"))) + return 0 + else: + summary = modal_eff.run(plan, run_dir) + (run_dir / "run.json").write_text(json.dumps(summary, indent=1, default=str)) + return cmd_eff_parse(argparse.Namespace(run_dir=str(run_dir))) + + +def cmd_eff_parse(args) -> int: + from .efficiency import metrics, parse + + run_dir = Path(args.run_dir) + parsed = parse.parse_run(run_dir) + (run_dir / "parsed.json").write_text(json.dumps(parsed, indent=1, default=str)) + summary = metrics.summarize(parsed) + (run_dir / "summary.json").write_text(json.dumps(summary, indent=1, default=str)) + print(json.dumps({"run": parsed["run_id"], "arm": parsed["arm"], "rounds": summary["rounds"], + "missing": summary["missing"]})) + return 0 if summary["rounds"] == summary["rounds_expected"] else 1 + + +def cmd_eff_report(args) -> int: + from .efficiency import report as eff_report + + runs = eff_report.load([Path(d) for d in args.run_dirs]) + Path(args.out).write_text(eff_report.markdown(runs, title=args.title)) + print(f"{len(runs)} parsed run(s) -> {args.out}") + return 0 + + def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(prog="python -m tools.eval") sub = p.add_subparsers(dest="cmd", required=True) @@ -143,6 +215,27 @@ def main(argv: list[str] | None = None) -> int: rep.add_argument("--html", default=None) rep.add_argument("--title", default="yeto RL capability evaluation") rep.set_defaults(fn=cmd_report) + from .efficiency.spec import ARMS, PRESETS + + ep = sub.add_parser("eff-plan") + ep.add_argument("preset", choices=sorted(PRESETS)) + ep.add_argument("--arm", choices=ARMS, default=None) + ep.set_defaults(fn=cmd_eff_plan) + er = sub.add_parser("eff-run") + er.add_argument("preset", choices=sorted(PRESETS)) + er.add_argument("--arm", choices=ARMS, required=True) + er.add_argument("--run-id", required=True) + er.add_argument("--out", default=str(EFF_OUT)) + er.add_argument("--dry-run", action="store_true") + er.set_defaults(fn=cmd_eff_run) + eparse = sub.add_parser("eff-parse") + eparse.add_argument("run_dir") + eparse.set_defaults(fn=cmd_eff_parse) + erep = sub.add_parser("eff-report") + erep.add_argument("run_dirs", nargs="+") + erep.add_argument("--out", required=True) + erep.add_argument("--title", default="yeto RL efficiency evaluation") + erep.set_defaults(fn=cmd_eff_report) args = p.parse_args(argv) return args.fn(args) diff --git a/tools/eval/efficiency/__init__.py b/tools/eval/efficiency/__init__.py new file mode 100644 index 00000000..54197bef --- /dev/null +++ b/tools/eval/efficiency/__init__.py @@ -0,0 +1,24 @@ +"""yeto RL efficiency evaluation (S26 E0.1 / E0.2). + +Eight arms per model preset: {bare, yeto} x {verl, miles} x {sync, async}. The +bare arms run the framework's own entry point from the same image (no yeto +code on their path); the yeto arms run the same framework through yeto. Every +arm is generated from one :class:`.spec.EffSpec`, so model, data, order seed, +reward, hyperparameters, max tokens, card type and count and image are equal +by construction (:func:`.spec.control_fingerprint`). + + python -m tools.eval eff-plan qwen3-0.6b [--arm ARM] + python -m tools.eval eff-run qwen3-0.6b --arm bare-verl-sync --run-id ID [--dry-run] + python -m tools.eval eff-parse RUN_DIR + python -m tools.eval eff-report RUN_DIR ... --out report.md + +* :mod:`.spec` -- presets, arms, the per-arm command plans; +* :mod:`.parse` -- framework logs and yeto tapes -> one per-round record format; +* :mod:`.metrics` -- per-run summary (throughput, segments, cost, off-policy); +* :mod:`.report` -- Markdown report with yeto-vs-bare overhead and range verdicts; +* :mod:`.modal_eff` -- the Modal harness (bare arms and yeto+verl); + yeto+Miles runs through the yeto launcher (``tools/eval/miles_local.sh``). + +Raw data is written to ``/raw/`` before anything is computed; +``parsed.json`` and the report are derived from it and can be rebuilt. +""" diff --git a/tools/eval/efficiency/eff_prep.py b/tools/eval/efficiency/eff_prep.py new file mode 100644 index 00000000..94fdee80 --- /dev/null +++ b/tools/eval/efficiency/eff_prep.py @@ -0,0 +1,65 @@ +"""Model and data preparation for the bare-framework arms (no yeto import). + +Run inside the container as ``python eff_prep.py ``. It downloads +the pinned model snapshot and the pinned dataset revision, then writes the +training prompts in the dataset's own row order: + +* ``/root/eff-data/train.parquet`` and ``test.parquet`` -- verl rows, the same + shape as ``yeto.rl.adapters.verl.island_entry.prepare_data`` (the yeto+verl + arm's data), so bare verl and yeto+verl read identical rows; +* ``/root/eff-data/train.jsonl`` -- Miles rows (``prompt`` = chat messages, + ``label``), for ``--input-key prompt --label-key label --apply-chat-template``; +* ``/root/eff-data/order.json`` -- sha256 of the label sequence (row order) and + the row count, so a report can show that two arms read the same rows. + +Shuffling is left to each framework (seeded by the spec's ``seed``), as in the +yeto arm of the same framework. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +DATA_ROOT = Path("/root/eff-data") +MODEL_ROOT = Path("/root/eff-models") +VAL_ROWS = 64 + + +def main(spec_path: str) -> int: + import pandas as pd + from huggingface_hub import hf_hub_download, snapshot_download + + spec = json.loads(Path(spec_path).read_text()) + model_dir = MODEL_ROOT / spec["model"].split("/")[-1] + snapshot_download(spec["model"], revision=spec["model_revision"], local_dir=str(model_dir)) + DATA_ROOT.mkdir(parents=True, exist_ok=True) + order = {} + for split in ("train", "test"): + src = hf_hub_download(spec["data"], f"{split}.parquet", repo_type="dataset", revision=spec["data_revision"]) + frame = pd.read_parquet(src) + rows = [{ + "data_source": "yeto/" + spec["data"], + "prompt": [{"role": m["role"], "content": m["content"]} for m in row["messages"]], + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": str(row["label"])}, + "extra_info": {"split": split, "index": int(i)}, + } for i, row in frame.iterrows()] + if split == "test": + rows = rows[:VAL_ROWS] + pd.DataFrame(rows).to_parquet(DATA_ROOT / f"{split}.parquet") + labels = [r["reward_model"]["ground_truth"] for r in rows] + order[split] = {"rows": len(rows), "label_sha256": hashlib.sha256("\n".join(labels).encode()).hexdigest()} + if split == "train": + with (DATA_ROOT / "train.jsonl").open("w") as fh: + for r in rows: + fh.write(json.dumps({"prompt": r["prompt"], "label": r["reward_model"]["ground_truth"]}) + "\n") + (DATA_ROOT / "order.json").write_text(json.dumps({**order, "model_dir": str(model_dir)}, indent=1)) + print(json.dumps({"model_dir": str(model_dir), **order})) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1])) diff --git a/tools/eval/efficiency/eff_reward.py b/tools/eval/efficiency/eff_reward.py new file mode 100644 index 00000000..4358028a --- /dev/null +++ b/tools/eval/efficiency/eff_reward.py @@ -0,0 +1,39 @@ +"""GSM8K exact-number reward for the bare-framework arms (no yeto import). + +The grading rule is a verbatim copy of ``yeto.rl.rewards.builtin.gsm8k_grade`` +(the reward of the yeto arms, ``--reward-function gsm8k_reward:score``); a CPU +test asserts both give the same value on the same cases. The file is copied +into the bare containers as ``/root/eff_reward.py``: + +* verl: ``reward.custom_reward_function.path=/root/eff_reward.py`` and + ``reward.custom_reward_function.name=compute_score``; +* Miles: ``--custom-rm-path eff_reward.score`` (``/root`` on ``PYTHONPATH``). +""" + +from __future__ import annotations + +import re + + +def _num(s): + s = str(s).replace(",", "") + m = re.findall(r"-?\d+(?:\.\d+)?", s) + return float(m[-1]) if m else None + + +def grade(response: str, label) -> float: + gold = _num(str(label).split("####")[-1]) if label is not None else None + text = response or "" + boxed = re.findall(r"\\boxed\{([^}]*)\}", text) + pred = _num(boxed[-1]) if boxed else _num(text[-200:]) + return 1.0 if (gold is not None and pred is not None and abs(pred - gold) < 1e-6) else 0.0 + + +def compute_score(data_source, solution_str, ground_truth, extra_info=None, **kwargs) -> float: + """verl custom reward signature.""" + return grade(solution_str, ground_truth) + + +async def score(args, sample, **kwargs) -> float: + """Miles custom reward signature (``--custom-rm-path``).""" + return grade(sample.response or "", sample.label) diff --git a/tools/eval/efficiency/metrics.py b/tools/eval/efficiency/metrics.py new file mode 100644 index 00000000..6fe298c0 --- /dev/null +++ b/tools/eval/efficiency/metrics.py @@ -0,0 +1,136 @@ +"""Per-run summary from ``parsed.json`` (the definitions every arm shares). + +* Steady window: the rounds after the first (the first carries engine start-up, + CUDA-graph capture and caches); ``steady_wall_s`` = the sum of their + ``round_s`` (step end minus previous step end). +* Throughput per card: steady tokens / steady wall / every card of the arm + (rollout and trainer cards both count). +* Segments: per-round medians over the steady window; ``unaccounted_s`` = round + minus the named segments (offload/onload, reward, advantage, orchestration). +* GPU: NVML samples inside the steady window; ``bubble_frac`` = the share of + card samples with utilization below :data:`BUBBLE_UTIL_PCT`. +* Cost: ``usd_per_card_hour`` from ``prices.json`` x cards x container seconds + (whole run) and x steady wall (steady state), per 1k trained samples. + +A value whose inputs are missing is ``None`` and its name is listed in +``missing``; nothing is filled in. +""" + +from __future__ import annotations + +import json +import statistics +from pathlib import Path +from typing import Any + +PRICES = Path(__file__).with_name("prices.json") +BUBBLE_UTIL_PCT = 5.0 +SEGMENTS = ("gen_s", "train_s", "weight_sync_s", "sampler_update_s", "wait_s") + + +def price(gpu: str, path: Path = PRICES) -> float: + table = json.loads(path.read_text()) + if gpu not in table: + raise KeyError(f"no price for GPU {gpu!r} in {path}") + return float(table[gpu]["usd_per_card_hour"]) + + +def _vals(rows: list[dict], key: str) -> list[float]: + return [float(r[key]) for r in rows if r.get(key) is not None] + + +def _median(values: list[float]) -> float | None: + return round(statistics.median(values), 4) if values else None + + +def _sum_all(rows: list[dict], key: str) -> float | None: + """Sum over rows, or None when any row lacks the value (a partial sum would mislead).""" + vals = [r.get(key) for r in rows] + return None if not vals or any(v is None for v in vals) else float(sum(vals)) + + +def summarize(parsed: dict[str, Any], *, prices_path: Path = PRICES) -> dict[str, Any]: + rounds = sorted(parsed["rounds"], key=lambda r: r["round"]) + cards = int(parsed["cards"]["total"]) + steady = rounds[1:] + s: dict[str, Any] = {"arm": parsed["arm"], "run_id": parsed["run_id"], "rounds": len(rounds), + "rounds_expected": parsed["spec"]["rounds"], "cards": cards, "gpu": parsed["gpu"], + "rc": parsed.get("rc")} + wall = _sum_all(steady, "round_s") + s["steady_wall_s"] = round(wall, 3) if wall is not None else None + for key, out in (("gen_tokens", "gen_tok_per_card_s"), ("train_tokens", "train_tok_per_card_s"), + ("samples", "samples_per_s")): + total = _sum_all(steady, key) + denom = (wall * (cards if key != "samples" else 1)) if wall else None + s[out] = round(total / denom, 3) if total is not None and denom else None + s["round_s_median"] = _median(_vals(steady, "round_s")) + s["round_s_range"] = [min(_vals(steady, "round_s")), max(_vals(steady, "round_s"))] if _vals(steady, "round_s") \ + else None + for seg in SEGMENTS: + s[f"{seg}_median"] = None if parsed["sources"].get(seg) in ("n/a",) else _median(_vals(steady, seg)) + named = [seg for seg in SEGMENTS if parsed["sources"].get(seg) != "n/a"] + rest = [r["round_s"] - sum(r[seg] for seg in named) for r in steady + if r.get("round_s") is not None and all(r.get(seg) is not None for seg in named)] + s["unaccounted_s_median"] = _median(rest) + s["segments_raw_median"] = _raw_medians(parsed.get("segments_raw") or []) + # reward against wall time and samples (every round; t from the first step end) + t0 = rounds[0].get("t_end_unix") if rounds else None + cum, curve = 0.0, [] + for r in rounds: + cum = cum + r["samples"] if r.get("samples") is not None and cum is not None else None + curve.append({"round": r["round"], "t_s": round(r["t_end_unix"] - t0, 3) + if r.get("t_end_unix") is not None and t0 is not None else None, + "samples": cum, "reward_mean": r.get("reward_mean")}) + s["reward_curve"] = curve + s["reward_last3_mean"] = _median(_vals(rounds[-3:], "reward_mean")) # median of the last three + # GPU (NVML samples inside the steady window) + lo = rounds[0].get("t_end_unix") if rounds else None + hi = rounds[-1].get("t_end_unix") if rounds else None + nv = [x for x in parsed.get("nvml") or [] if lo is not None and hi is not None and lo <= x["t_unix"] <= hi] + util = [x["util"] for x in nv if x.get("util") is not None] + s["nvml_samples"] = len(util) + s["nvml_interval_s"] = parsed.get("nvml_interval_s") + s["gpu_util_mean"] = round(statistics.fmean(util), 2) if util else None + s["bubble_frac"] = round(sum(u < BUBBLE_UTIL_PCT for u in util) / len(util), 4) if util else None + mem = [x["mem_mb"] for x in parsed.get("nvml") or [] if x.get("mem_mb") is not None] + s["gpu_mem_peak_mb"] = max(mem) if mem else None + # cost + rate = price(parsed["gpu"], prices_path) + s["usd_per_card_hour"] = rate + start, end = parsed.get("container_start_unix"), parsed.get("container_end_unix") + total_samples = _sum_all(rounds, "samples") + if start and end: + s["usd_run"] = round(rate * cards * (end - start) / 3600, 4) + s["usd_per_1k_samples_run"] = round(s["usd_run"] / total_samples * 1000, 4) if total_samples else None + else: + s["usd_run"] = s["usd_per_1k_samples_run"] = None + steady_samples = _sum_all(steady, "samples") + s["usd_per_1k_samples_steady"] = round(rate * cards * wall / 3600 / steady_samples * 1000, 4) \ + if wall and steady_samples else None + # off-policy + s["is_clip_frac_mean"] = round(statistics.fmean(_vals(rounds, "is_clip_frac")), 5) \ + if _vals(rounds, "is_clip_frac") else None + dropped, generated = _vals(rounds, "dropped_samples"), _vals(rounds, "generated_samples") + s["dropped_total"] = max(dropped) if dropped and "count/" in parsed["sources"].get("dropped_samples", "") \ + else (sum(dropped) if dropped else None) # verl's fully_async counters are cumulative + s["drop_frac"] = round(s["dropped_total"] / max(generated), 4) if s["dropped_total"] is not None and generated \ + and max(generated) else None + s["version_lag"] = [r.get("version_lag") for r in rounds if r.get("version_lag") is not None] or None + s["sources"] = parsed["sources"] + na = {k for k, v in parsed["sources"].items() if v == "n/a"} + s["not_applicable"] = sorted(k for k in s if s[k] is None and _field_of(k) in na) + s["missing"] = sorted(k for k, v in s.items() if v is None and k != "rc" and k not in s["not_applicable"]) + return s + + +# summary key -> the round field it is computed from (for the not-applicable list) +_DERIVED_FROM = {"dropped_total": "dropped_samples", "drop_frac": "dropped_samples"} + + +def _field_of(key: str) -> str: + return _DERIVED_FROM.get(key, key.removesuffix("_median")) + + +def _raw_medians(rows: list[dict[str, float]]) -> dict[str, float]: + keys = sorted({k for r in rows[1:] for k in r}) + return {k: _median([float(r[k]) for r in rows[1:] if r.get(k) is not None]) for k in keys} diff --git a/tools/eval/efficiency/modal_eff.py b/tools/eval/efficiency/modal_eff.py new file mode 100644 index 00000000..0bcf5719 --- /dev/null +++ b/tools/eval/efficiency/modal_eff.py @@ -0,0 +1,179 @@ +"""Modal harness of the efficiency arms that run in one container: the four bare +arms and yeto+verl (yeto+Miles runs through the yeto launcher, ``miles_local.sh``). + +One GPU container per run, ``:``, in the framework's image (the +same image the yeto arm of that framework uses). The container: + +1. checks every visible GPU is the asked type (``modal_verl._gpu``); +2. samples NVML every second into ``nvml.csv`` (index, util, memory, power); +3. bare: writes the spec, ``eff_reward.py`` and ``eff_prep.py`` under + ``/root/eff`` and runs the arm's bash script (prep, then the framework entry); + yeto+verl: starts the island Ray and runs ``island_entry`` (as T-EVAL E1); +4. records ``meta.json`` (start/end unix time, rc, GPU names, argv/script) and + copies its outputs into the results Volume ``/arm/`` before returning. + +The local side spawns the call in a detached app, waits, pulls ``/`` into +``/raw/`` and stops only its own app (``yeto-eff-``). +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from .. import modal_verl as mv +from .spec import EFF_DIR, OUT_DIR + +HERE = Path(__file__).resolve().parent +APP_PREFIX = "yeto-eff-" +NVML_QUERY = "timestamp,index,name,utilization.gpu,memory.used,memory.total,power.draw" +NVML_INTERVAL_MS = 1000 + + +# ====================================================================== in-container +def _nvml(path: Path) -> subprocess.Popen: + return subprocess.Popen(["bash", "-c", f"nvidia-smi --query-gpu={NVML_QUERY} --format=csv " + f"-lms {NVML_INTERVAL_MS} >> {path}"]) + + +def _run_logged(argv: list[str], log: Path, timeout: float, **kwargs) -> int: + """Run ``argv``; every output line is written to ``log`` behind `` `` + (the step end times the parsers read). The process is killed at ``timeout``.""" + deadline = time.time() + timeout + with log.open("a") as fh, subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + errors="replace", **kwargs) as proc: + for line in proc.stdout: + fh.write(f"{time.time():.3f} {line}") + fh.flush() + if time.time() > deadline: + proc.kill() + fh.write(f"{time.time():.3f} [eff harness] killed at the {timeout:.0f} s timeout\n") + break + return proc.wait() + + +def arm(run: str, plan: dict) -> dict: + """One run of one arm (see the module doc).""" + out = Path(mv.RESULTS) / run / "arm" + out.mkdir(parents=True, exist_ok=True) + meta: dict[str, Any] = {"arm": plan["arm"], "container": os.environ.get("MODAL_TASK_ID"), + "cards": plan["cards"]["total"]} + meta["gpu"] = mv._gpu(out, {"gpu": plan["gpu"]}) + nvml = _nvml(out / "nvml.csv") + meta["start_unix"] = time.time() + copies = {"eff-out": Path(OUT_DIR), "yeto-output": Path("/root/yeto-output")} + budget = plan["spec"]["timeout_s"] - 300 # leaves time to copy and commit + log = out / "main.log" + try: + if plan["system"] == "bare": + eff = Path(EFF_DIR) + eff.mkdir(parents=True, exist_ok=True) + for name, text in plan["files"].items(): + (eff / name).write_text(text) + (eff / "spec.json").write_text(json.dumps(plan["spec"])) + (out / "script.sh").write_text(plan["script"]) + meta["rc"] = _run_logged(["bash", str(out / "script.sh")], log, budget) + else: + sys.path.insert(0, mv.WORKDIR) + env = mv._island_env({"gpu": plan["gpu"]}, 0) + ray = str(Path(mv.VERL_PY).with_name("ray")) + _run_logged([ray, "start", "--head", "--node-ip-address=127.0.0.1", "--port=6379", "--num-cpus=32", + f"--num-gpus={plan['cards']['total']}", "--include-dashboard=false", + "--temp-dir=/root/eval-ray"], log, 300, env=env) + meta["argv"] = [mv.VERL_PY, *plan["argv"]] + meta["rc"] = _run_logged(meta["argv"], log, budget, env=env, cwd="/root") + except BaseException as exc: # noqa: BLE001 - recorded, then re-raised + meta["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + meta["end_unix"] = time.time() + nvml.terminate() + mv._finish(out, meta, meta["start_unix"], copies) + return meta + + +# ====================================================================== app +def build_app(run: str, plan: dict): + import modal + + sys.path.insert(0, str(mv.REPO)) + from modal._vendor import cloudpickle + + # the container functions travel by value: `tools` is not importable in the images + cloudpickle.register_pickle_by_value(sys.modules[__name__]) + cloudpickle.register_pickle_by_value(mv) + from yeto.rl.adapters.miles.pins import MILES_NEXT_IMAGE + from yeto.rl.adapters.verl.image import modal_image + + if plan["image"] == "verl": + image = modal_image(modal).env({"PYTHONUNBUFFERED": "1"}) + if plan["system"] == "yeto": + image = image.add_local_dir(str(mv.REPO), mv.WORKDIR, copy=False, ignore=MOUNT_IGNORE) + else: + image = (modal.Image.from_registry(MILES_NEXT_IMAGE.removeprefix("docker:")) + .env({"HOME": "/root", "PYTHONUNBUFFERED": "1"})) + app = modal.App(APP_PREFIX + run) + results = modal.Volume.from_name(mv.RESULTS_VOLUME, create_if_missing=True, version=2) + gpu = f"{plan['gpu']}:{plan['cards']['total']}" + fn = app.function(image=image, gpu=gpu, cpu=8.0 * plan["cards"]["total"], + memory=49152 * plan["cards"]["total"], timeout=plan["spec"]["timeout_s"], retries=0, + volumes={mv.RESULTS: results}, single_use_containers=True, name="arm", serialized=True)(arm) + return app, fn + + +MOUNT_IGNORE = [".git", ".venv", ".claude", "**/__pycache__", "**/*.pyc", "s1-runs", "syncer/target", + "evidence*", "openspec", "docs", "tutorial", "data"] + + +def bare_files() -> dict[str, str]: + return {name: (HERE / name).read_text() for name in ("eff_reward.py", "eff_prep.py")} + + +def run(plan: dict, run_dir: Path) -> dict[str, Any]: + """Run one arm end to end (spawn, wait, pull, stop own app).""" + import modal + + run_id = plan["run_id"] + raw = run_dir / "raw" + raw.mkdir(parents=True, exist_ok=True) + log = run_dir / "driver.log" + if plan["system"] == "bare": + plan = {**plan, "files": bare_files()} + app, fn = build_app(run_id, plan) + began = time.time() + with modal.enable_output(), app.run(detach=True): # a killed driver never cancels a spawned call + app_id = app.app_id + (run_dir / "app_id.txt").write_text(f"{app_id}\n") + call = fn.spawn(run_id, plan) + mv._log(log, f"spawned arm {plan['arm']} app {APP_PREFIX + run_id}") + results = mv._wait_calls({"arm": call}, began + plan["spec"]["timeout_s"] + 600, log) + pulled = mv._modal_cli("volume", "get", "--force", mv.RESULTS_VOLUME, run_id, str(raw), timeout=1800) + stopped = mv._modal_cli("app", "stop", "-y", app_id, timeout=120) + summary = {"calls": results, "wall_s": round(time.time() - began, 1), "pull_rc": pulled.returncode, + "stop_rc": stopped.returncode} + (run_dir / "driver.json").write_text(json.dumps(summary, indent=1, default=str)) + return summary + + +# ====================================================================== yeto+Miles (launcher) +def run_miles_launcher(plan: dict, run_dir: Path, *, dry_run: bool = False) -> dict[str, Any]: + """The yeto+Miles arm: ``miles_local.sh`` with the plan's launcher arguments + (the T-EVAL Miles path; it pulls launch log, tapes and NVML into ``raw/``).""" + from .. import miles + + raw = run_dir / "raw" + env = dict(os.environ, P=plan["run_id"], R=str(raw), REPO=str(miles.REPO), PY=miles.PY, SKY=miles.SKY, + STOREVOL=miles.STORE_VOLUME, TAPEVOL=miles.TAPE_VOLUME, HARD=str(plan["spec"]["timeout_s"]), + HOME="/home/michael", ARGS=plan["launcher_args"], ISLANDS="1") + if dry_run: + env["DRY"] = "1" + log = run_dir / ("dry-run.log" if dry_run else "driver.log") + with log.open("w") as fh: + rc = subprocess.run(["bash", str(miles.HERE / "miles_local.sh")], env=env, stdout=fh, + stderr=subprocess.STDOUT, cwd="/tmp").returncode + return {"script": "miles_local.sh", "rc": rc, "log": str(log)} diff --git a/tools/eval/efficiency/parse.py b/tools/eval/efficiency/parse.py new file mode 100644 index 00000000..167d0bab --- /dev/null +++ b/tools/eval/efficiency/parse.py @@ -0,0 +1,420 @@ +"""Framework logs and yeto tapes -> one per-round record format. + +Every parser returns a :class:`Parsed`: ``rounds`` (one dict per optimizer +step, the fields of :data:`ROUND_FIELDS`), ``sources`` (per field: the log key +it came from, ``"derived: ..."`` when it is computed from logged values by an +exact identity, ``MISSING`` when no source in the raw data carries it, or +``NA`` when the field does not apply to the arm), and ``segments_raw`` (every +native timing per round, for the overhead table). + +A field that a log does not carry stays ``None``; nothing is estimated. + +Formats read (each one is cited where it is parsed): + +* verl ``FileLogger`` JSONL (``VERL_FILE_LOGGER_PATH``): one + ``{"step": N, "data": {...}}`` per step (fork acad9875 ``verl/utils/tracking.py`` + ``FileLogger.log``); keys from ``metric_utils.compute_timing_metrics`` / + ``compute_throughout_metrics`` (``timing_s/*``, ``perf/*``) and the fully_async + trainer (``fully_async/*``, ``timing_s/param_sync``, ``timing_s/wait_for_enough_samples``); +* verl console lines ``step:N - key:value - ...`` (``LocalLogger``, + ``concat_dict_to_str``) behind the harness's `` `` line prefix, for the + step end time; +* Miles ``perf N: {python dict}`` log lines: rollout side + (``miles/ray/rollout/metrics.py:93``, keys ``rollout/*``, ``perf/rollout_time``, + ``perf/tokens_per_gpu_per_sec``) and train side + (``miles/utils/train_metric_utils.py:56``, keys ``perf/_time``, + ``perf/actor_train_tok_per_s``); the line's ``[YYYY-mm-dd HH:MM:SS.mmm`` stamp + (Miles' logger format) gives the time; +* yeto event tapes (JSONL ``{"event": ...}``): ``rl_round_wall``, + ``rl_local_round``, ``rl_round_trained``, ``rl_timeline_span``, + ``rl_driver_phase``, ``rl_rollout_carry_over``, ``rl_resource_sample``; +* NVML CSV (``nvidia-smi --query-gpu=timestamp,index,name,utilization.gpu, + memory.used,memory.total,power.draw --format=csv``), with or without a + ``,`` prefix per line (``miles_local.sh`` polls every 20 s). +""" + +from __future__ import annotations + +import ast +import csv +import datetime as _dt +import io +import json +import re +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +MISSING = "missing" +NA = "n/a" +ROUND_FIELDS = ( + "round", "t_end_unix", "round_s", "gen_s", "train_s", "weight_sync_s", "sampler_update_s", "wait_s", + "gen_tokens", "train_tokens", "samples", "reward_mean", "is_clip_frac", "dropped_samples", + "generated_samples", "version_lag", +) +ANSI = re.compile(r"\x1b\[[0-9;]*m") + + +@dataclass +class Parsed: + rounds: list[dict[str, Any]] = field(default_factory=list) + sources: dict[str, str] = field(default_factory=dict) + segments_raw: list[dict[str, float]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return {"rounds": self.rounds, "sources": self.sources, "segments_raw": self.segments_raw} + + +def _first(data: dict, keys: Iterable[str]) -> tuple[Any, str | None]: + for key in keys: + if data.get(key) is not None: + return data[key], key + return None, None + + +def _finish(parsed: Parsed, na: Iterable[str] = ()) -> Parsed: + """Fill ``sources`` for fields no round carries (missing or not applicable).""" + na = set(na) + parsed.sources["round"] = "step index (verl global step; Miles / yeto rollout id)" + for name in ROUND_FIELDS[1:]: + if name in na: + parsed.sources[name] = NA + for r in parsed.rounds: + r[name] = None + elif name not in parsed.sources or all(r.get(name) is None for r in parsed.rounds): + parsed.sources[name] = MISSING + return parsed + + +def _record(rnd: int) -> dict[str, Any]: + return {name: None for name in ROUND_FIELDS} | {"round": rnd} + + +def _set(parsed: Parsed, rec: dict, name: str, value: Any, source: str | None) -> None: + if value is None or source is None: + return + rec[name] = value + parsed.sources.setdefault(name, source) + + +def _step_ends(log_text: str, pattern: re.Pattern) -> dict[int, float]: + """Round -> unix time of its log line (lines carry a `` `` prefix).""" + out: dict[int, float] = {} + for line in log_text.splitlines(): + m = pattern.search(ANSI.sub("", line)) + if m: + out[int(m.group(2))] = float(m.group(1)) + return out + + +def _round_s_from_ends(parsed: Parsed) -> None: + """round_s = time between consecutive step ends (one definition for every arm).""" + prev = None + for r in parsed.rounds: + t = r.get("t_end_unix") + if t is not None and prev is not None: + r["round_s"] = round(t - prev, 3) + parsed.sources.setdefault("round_s", "derived: step end - previous step end") + prev = t + + +# ====================================================================== verl +VERL_STEP_LINE = re.compile(r"^(\d+(?:\.\d+)?) .*?\bstep:(\d+) - ") + + +def parse_verl(metrics_jsonl: str, log_text: str = "", *, samples_per_round: int, trainer_cards: int, + mode: str) -> Parsed: + """verl FileLogger rows (+ console step lines for end times).""" + p = Parsed() + ends = _step_ends(log_text, VERL_STEP_LINE) + for line in metrics_jsonl.splitlines(): + if not line.strip(): + continue + row = json.loads(line) + d, step = row.get("data") or {}, int(row["step"]) + rec = _record(step) + _set(p, rec, "t_end_unix", ends.get(step), "console step line (harness time prefix)") + gen, key = _first(d, ("timing_s/gen",)) + _set(p, rec, "gen_s", gen, key) + parts = [k for k in ("timing_s/old_log_prob", "timing_s/update_actor") if d.get(k) is not None] + if parts: + _set(p, rec, "train_s", sum(float(d[k]) for k in parts), " + ".join(parts)) + sync, key = _first(d, ("timing_s/update_weights", "timing_s/param_sync")) + _set(p, rec, "weight_sync_s", sync, key) + wait, key = _first(d, ("timing_s/wait_for_enough_samples",)) + _set(p, rec, "wait_s", wait, key) + if d.get("response_length/mean") is not None: # every step trains exactly samples_per_round responses + _set(p, rec, "gen_tokens", float(d["response_length/mean"]) * samples_per_round, + f"derived: response_length/mean x {samples_per_round} responses") + _set(p, rec, "samples", samples_per_round, "config: samples per step (verl trains a fixed batch)") + tokens, key = _first(d, ("perf/total_num_tokens",)) + if tokens is None and d.get("global_seqlen/mean") is not None: + tokens, key = float(d["global_seqlen/mean"]) * trainer_cards, \ + f"derived: global_seqlen/mean x {trainer_cards} trainer ranks" + _set(p, rec, "train_tokens", tokens, key) + reward, key = _first(d, ("critic/score/mean", "critic/rewards/mean")) + _set(p, rec, "reward_mean", reward, key) + clip, key = _first(d, ("rollout_corr/rollout_is_ratio_fraction_high", "yeto_mismatch/tis_clipfrac")) + _set(p, rec, "is_clip_frac", clip, key) + dropped, key = _first(d, ("fully_async/count/dropped_stale_samples",)) + _set(p, rec, "dropped_samples", dropped, key) + generated, key = _first(d, ("fully_async/count/total_generated_samples",)) + _set(p, rec, "generated_samples", generated, key) + p.rounds.append(rec) + p.segments_raw.append({k: float(v) for k, v in d.items() if k.startswith("timing_s/") + and isinstance(v, (int, float))}) + p.rounds.sort(key=lambda r: r["round"]) + _round_s_from_ends(p) + na = {"sampler_update_s"} + if mode == "sync": + na |= {"wait_s", "dropped_samples", "generated_samples", "version_lag"} + return _finish(p, na) + + +# ====================================================================== Miles +MILES_DICT_LINE = re.compile(r"\b(perf|step|rollout) (\d+): (\{.*\})\s*$") +MILES_STAMP = re.compile(r"\[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d+)") + + +def _miles_stamp(line: str) -> float | None: + m = MILES_STAMP.search(line) + if not m: + return None + return _dt.datetime.strptime(m.group(1), "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=_dt.timezone.utc).timestamp() + + +def miles_dicts(log_text: str) -> dict[int, dict[str, Any]]: + """Rollout id -> {"rollout": {...}, "train": {...}, "step": {...}, "train_t": unix}.""" + out: dict[int, dict[str, Any]] = defaultdict(dict) + for raw in log_text.splitlines(): + line = ANSI.sub("", raw) + m = MILES_DICT_LINE.search(line) + if not m: + continue + try: + data = ast.literal_eval(m.group(3)) + except (ValueError, SyntaxError): + continue + rid = int(m.group(2)) + if m.group(1) == "perf" and ("perf/rollout_time" in data or any(k.startswith("rollout/") for k in data)): + out[rid].setdefault("rollout", {}).update(data) + elif m.group(1) == "perf": + out[rid].setdefault("train", {}).update(data) + out[rid]["train_t"] = _miles_stamp(line) + else: + out[rid].setdefault("step", {}).update(data) + return dict(out) + + +def parse_miles(log_text: str, *, rollout_cards: int, mode: str) -> Parsed: + p = Parsed() + for rid, parts in sorted(miles_dicts(log_text).items()): + ro, tr, st = parts.get("rollout", {}), parts.get("train", {}), parts.get("step", {}) + merged = {**ro, **tr, **st} + rec = _record(rid) + _set(p, rec, "t_end_unix", parts.get("train_t"), "Miles train perf line time stamp") + gen, key = _first(ro, ("perf/rollout_time",)) + _set(p, rec, "gen_s", gen, key) + train, key = _first(tr, ("perf/train_time",)) + _set(p, rec, "train_s", train, key) + sync, key = _first(tr, ("perf/update_weights_time",)) + _set(p, rec, "weight_sync_s", sync, key) + if mode == "async": + wait, key = _first(tr, ("perf/train_wait_time",)) + _set(p, rec, "wait_s", wait, key) + if ro.get("perf/tokens_per_gpu_per_sec") is not None and gen is not None: + _set(p, rec, "gen_tokens", float(ro["perf/tokens_per_gpu_per_sec"]) * float(gen) * rollout_cards, + f"derived: perf/tokens_per_gpu_per_sec x perf/rollout_time x {rollout_cards} rollout cards") + if tr.get("perf/actor_train_tok_per_s") is not None and tr.get("perf/actor_train_time") is not None: + _set(p, rec, "train_tokens", float(tr["perf/actor_train_tok_per_s"]) * float(tr["perf/actor_train_time"]), + "derived: perf/actor_train_tok_per_s x perf/actor_train_time") + samples, key = _first(merged, ("rollout/num_training_samples", "num_training_samples")) + _set(p, rec, "samples", samples, key) + reward, key = _first(merged, ("rollout/raw_reward", "rollout/episode_raw_reward", "rollout/rewards")) + _set(p, rec, "reward_mean", reward, key) + clip, key = _first(merged, ("train/tis_clipfrac", "train/tis_clip_frac", "tis_clipfrac")) + _set(p, rec, "is_clip_frac", clip, key) + lag = {k: v for k, v in merged.items() if "staleness" in k or "weight_version" in k} + _set(p, rec, "version_lag", lag or None, "Miles keys containing 'staleness' / 'weight_version'") + p.rounds.append(rec) + p.segments_raw.append({k: float(v) for k, v in merged.items() + if k.startswith("perf/") and k.endswith("_time") and isinstance(v, (int, float))}) + _round_s_from_ends(p) + na = {"sampler_update_s"} + if mode == "sync": + na |= {"wait_s", "dropped_samples", "generated_samples", "version_lag"} + return _finish(p, na) + + +# ====================================================================== yeto tape +def tape_rows(text: str) -> list[dict[str, Any]]: + rows = [] + for line in text.splitlines(): + line = line.strip() + start = line.find("{") + if start < 0: + continue + try: + row = json.loads(line[start:]) + except json.JSONDecodeError: + continue + if isinstance(row, dict) and row.get("event"): + rows.append(row) + return rows + + +# yeto task / phase names -> canonical segment (others stay in segments_raw only) +YETO_SEGMENT = {"generate": "gen_s", "train": "train_s", "publish": "weight_sync_s"} + + +def parse_yeto(events: list[dict[str, Any]], *, mode: str, framework_parsed: Parsed | None = None, + framework_round_offset: int = 0) -> Parsed: + """yeto tape -> rounds; ``framework_parsed`` (the framework's own log in the + same run, e.g. Miles perf lines) fills fields the tape does not carry. Its + round ``rollout_id + framework_round_offset`` is the tape's ``rollout_id``: + 0 for Miles (perf ids are rollout ids), 1 for verl (FileLogger step = + global step, 1-based; T-EVAL eval-e1-verl-c: steps 1..4, rollout ids 0..3).""" + p = Parsed() + by = defaultdict(list) + for e in events: + by[e["event"]].append(e) + walls = {int(e["rollout_id"]): e for e in by["rl_round_wall"] if e.get("rollout_id") is not None} + trained = {int(e["rollout_id"]): e for e in by["rl_round_trained"] if e.get("rollout_id") is not None} + local = sorted(by["rl_local_round"], key=lambda e: float(e.get("time_unix") or 0)) + spans: dict[int, dict[str, float]] = defaultdict(lambda: defaultdict(float)) + for s in by["rl_timeline_span"]: + if s.get("rollout_id") is None or s.get("start") is None or s.get("end") is None: + continue + name = "wait" if s.get("kind") == "wait" else str(s.get("task")) + spans[int(s["rollout_id"])][name] += float(s["end"]) - float(s["start"]) + phases = _driver_phase_durations(by["rl_driver_phase"]) + drops = defaultdict(float) + for e in by["rl_rollout_carry_over"]: + if e.get("rollout_id") is not None and e.get("over_age_discarded_groups") is not None: + drops[int(e["rollout_id"])] += float(e["over_age_discarded_groups"]) + rounds = sorted(set(walls) | set(trained)) + fw = {r["round"]: r for r in (framework_parsed.rounds if framework_parsed else [])} + for i, rid in enumerate(rounds): + rec = _record(rid) + w, t = walls.get(rid, {}), trained.get(rid, {}) + lr = local[i] if i < len(local) else {} + _set(p, rec, "t_end_unix", w.get("time_unix"), "rl_round_wall.time_unix") + seg = dict(spans.get(rid) or phases.get(rid) or {}) + src = "rl_timeline_span" if rid in spans else "rl_driver_phase (marker to next marker)" + for task, name in YETO_SEGMENT.items(): + _set(p, rec, name, seg.get(task), f"{src} {task}") + if mode == "async": + _set(p, rec, "wait_s", seg.get("wait"), f"{src} kind=wait") + _set(p, rec, "gen_tokens", lr.get("action_tokens"), "rl_local_round.action_tokens") + _set(p, rec, "samples", t.get("trained_samples"), "rl_round_trained.trained_samples") + _set(p, rec, "reward_mean", lr.get("reward_mean"), "rl_local_round.reward_mean") + _set(p, rec, "is_clip_frac", (t.get("mismatch") or {}).get("tis_clipfrac"), "rl_round_trained.mismatch.tis_clipfrac") + if mode == "async": + _set(p, rec, "dropped_samples", drops.get(rid), "rl_rollout_carry_over.over_age_discarded_groups (groups)") + f = fw.get(rid + framework_round_offset, {}) + for name in ROUND_FIELDS: # what the tape lacks, from the framework's own log of the same run + if rec[name] is None and f.get(name) is not None and name not in ("round", "t_end_unix", "round_s"): + _set(p, rec, name, f[name], "framework log: " + framework_parsed.sources[name]) + p.rounds.append(rec) + p.segments_raw.append({k: round(v, 3) for k, v in seg.items()}) + _round_s_from_ends(p) + na = {"sampler_update_s"} # single-island arms: no separate sampler island + if mode == "sync": + na |= {"wait_s", "dropped_samples", "generated_samples", "version_lag"} + return _finish(p, na) + + +def _driver_phase_durations(markers: list[dict[str, Any]]) -> dict[int, dict[str, float]]: + """rl_driver_phase markers (one per phase start) -> rollout id -> phase -> seconds + (marker to the next marker; a marker without rollout_id belongs to the round in progress).""" + out: dict[int, dict[str, float]] = defaultdict(lambda: defaultdict(float)) + rows = sorted(markers, key=lambda e: float(e.get("time_unix") or 0)) + current = 0 + for a, b in zip(rows, rows[1:]): + if a.get("rollout_id") is not None: + current = int(a["rollout_id"]) + out[current][str(a.get("phase"))] += float(b["time_unix"]) - float(a["time_unix"]) + return out + + +# ====================================================================== NVML +def parse_nvml(text: str) -> list[dict[str, Any]]: + """Rows {t_unix, key (container:index), util, mem_mb, mem_total_mb, power_w}; header lines skipped.""" + rows = [] + for parts in csv.reader(io.StringIO(text)): + parts = [x.strip() for x in parts] + if not parts or parts[0].startswith("timestamp") or "timestamp" in parts: + continue + container = "" + if not re.match(r"\d{4}/\d\d/\d\d", parts[0]): + container, parts = parts[0], parts[1:] + try: + t = _dt.datetime.strptime(parts[0], "%Y/%m/%d %H:%M:%S.%f").replace(tzinfo=_dt.timezone.utc).timestamp() + except (ValueError, IndexError): + continue + num = [re.sub(r"[^\d.]", "", x) for x in parts] + if len(parts) >= 7: # timestamp,index,name,util,mem.used,mem.total,power + index, util, mem, total, power = parts[1], num[3], num[4], num[5], num[6] + else: # timestamp,name,util,mem.used,power (T-EVAL format, no index) + index, util, mem, total, power = "0", num[2], num[3], "", num[4] if len(num) > 4 else "" + rows.append({"t_unix": t, "key": f"{container}:{index}", "util": float(util) if util else None, + "mem_mb": float(mem) if mem else None, "mem_total_mb": float(total) if total else None, + "power_w": float(power) if power else None}) + return rows + + +# ====================================================================== run directory +def _read(path: Path) -> str: + return path.read_text(errors="replace") if path.is_file() else "" + + +def _tape_events(raw: Path) -> list[dict[str, Any]]: + seen, rows = set(), [] + for path in sorted(raw.rglob("*.jsonl")): + if "verl-" in path.name: + continue + for row in tape_rows(_read(path)): + key = json.dumps(row, sort_keys=True, default=str) # the same event pulled from two sources + if key not in seen: + seen.add(key) + rows.append(row) + return rows + + +def parse_run(run_dir: Path) -> dict[str, Any]: + """Parse a run directory (``arm.json`` + ``raw/``) into ``parsed.json`` content.""" + run_dir = Path(run_dir) + plan = json.loads((run_dir / "arm.json").read_text()) + raw = run_dir / "raw" + mode, spec, c = plan["mode"], plan["spec"], plan["cards"] + arm_dir = next(iter(sorted(raw.glob("*/arm"))), raw / "arm") + nvml_text = _read(arm_dir / "nvml.csv") or _read(raw / "nvml.csv") + meta = json.loads(_read(arm_dir / "meta.json") or "{}") + if plan["framework"] == "verl": + metrics = next((p for p in sorted(raw.rglob("verl-metrics.jsonl"))), None) \ + or next((p for p in sorted(raw.rglob("verl-file-logger-*.jsonl"))), None) + fw = parse_verl(_read(metrics) if metrics else "", _read(arm_dir / "main.log"), + samples_per_round=spec["groups_per_round"] * spec["samples_per_group"], + trainer_cards=c["trainer"], mode=mode) + else: + log = _read(arm_dir / "main.log") or _read(raw / "launch.ts.log") + fw = parse_miles(log, rollout_cards=c["rollout"], mode=mode) + parsed = parse_yeto(_tape_events(raw), mode=mode, framework_parsed=fw, + framework_round_offset=1 if plan["framework"] == "verl" else 0) \ + if plan["system"] == "yeto" else fw + start = meta.get("start_unix") or _utc_file(raw / "start_utc.txt") + end = meta.get("end_unix") or _utc_file(raw / "end_utc.txt") + return {"arm": plan["arm"], "run_id": plan["run_id"], "fingerprint": plan["fingerprint"], "mode": mode, + "framework": plan["framework"], "system": plan["system"], "cards": c, "gpu": plan["gpu"], + "spec": spec, "container_start_unix": start, "container_end_unix": end, "rc": meta.get("rc"), + "framework_log": fw.to_dict() if plan["system"] == "yeto" else None, + "nvml": parse_nvml(nvml_text), "nvml_interval_s": 20.0 if plan["harness"] == "miles_local" else 1.0, + **parsed.to_dict()} + + +def _utc_file(path: Path) -> float | None: + text = _read(path).strip() + if not text: + return None + return _dt.datetime.strptime(text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=_dt.timezone.utc).timestamp() diff --git a/tools/eval/efficiency/prices.json b/tools/eval/efficiency/prices.json new file mode 100644 index 00000000..75edc4ee --- /dev/null +++ b/tools/eval/efficiency/prices.json @@ -0,0 +1,6 @@ +{ + "_note": "USD per card-hour, all-in container estimate (GPU + the CPU and memory the harness requests). Modal list prices read 2026-10-20 for the GPU part; CPU/memory added as in tools/eval/cells.py. Estimates, not a bill: replace with the ledger's rate (infra-drafts/gpu-spend.md) when it differs.", + "A10G": {"usd_per_card_hour": 2.0, "gpu_list_usd_per_hour": 1.10}, + "L4": {"usd_per_card_hour": 1.6, "gpu_list_usd_per_hour": 0.80}, + "H100!": {"usd_per_card_hour": 5.3, "gpu_list_usd_per_hour": 3.95} +} diff --git a/tools/eval/efficiency/report.py b/tools/eval/efficiency/report.py new file mode 100644 index 00000000..38c6a9f0 --- /dev/null +++ b/tools/eval/efficiency/report.py @@ -0,0 +1,205 @@ +"""Markdown report of a set of efficiency runs (``parsed.json`` in each run dir). + +Sections: the shared spec and its fingerprint; one row per run; per-arm +aggregates (median and min-max over repeats); yeto vs bare per framework and +mode, metric by metric, with a range verdict; yeto's own segments; reward +curves; and the fields each arm could not measure. + +Range verdict (repeats are not bit-reproducible on H100, so no single-run +comparison): with >= :data:`MIN_REPEATS` runs on both sides, yeto is "worse" +or "better" only when its min-max range and bare's do not overlap; otherwise +"within run-to-run range". Fewer runs, or a missing value, is said as such. +""" + +from __future__ import annotations + +import json +import statistics +from collections import defaultdict +from pathlib import Path +from typing import Any + +from .metrics import summarize +from .spec import ARMS, FRAMEWORKS, MODES, EffSpec, verl_knob_diff + +MIN_REPEATS = 2 +# metric -> (label, True when lower is better) +COMPARED = { + "round_s_median": ("round time (s)", True), + "gen_s_median": ("generation (s/round)", True), + "train_s_median": ("training (s/round)", True), + "weight_sync_s_median": ("weight sync trainer->rollout (s/round)", True), + "sampler_update_s_median": ("update sampler weights (s/round)", True), + "wait_s_median": ("wait (s/round)", True), + "unaccounted_s_median": ("other, not in the segments above (s/round)", True), + "gen_tok_per_card_s": ("generated tokens / card / s", False), + "train_tok_per_card_s": ("trained tokens / card / s", False), + "samples_per_s": ("trained samples / s", False), + "gpu_util_mean": ("GPU utilization mean (%)", False), + "bubble_frac": ("idle bubble share (util < 5 %)", True), + "gpu_mem_peak_mb": ("GPU memory peak (MiB)", True), + "usd_per_1k_samples_steady": ("USD / 1k samples (steady)", True), + "usd_per_1k_samples_run": ("USD / 1k samples (whole container run)", True), +} + + +def load(run_dirs: list[Path]) -> list[dict[str, Any]]: + out = [] + for d in run_dirs: + path = Path(d) / "parsed.json" + if path.is_file(): + out.append(json.loads(path.read_text())) + return out + + +def _fmt(v: Any) -> str: + if v is None: + return "missing" + if isinstance(v, float): + return f"{v:.4g}" + return str(v) + + +def _range(values: list[float]) -> str: + return "missing" if not values else f"{_fmt(statistics.median(values))} [{_fmt(min(values))}, {_fmt(max(values))}]" + + +def verdict(bare: list[float | None], yeto: list[float | None], lower_is_better: bool) -> str: + if any(v is None for v in bare + yeto) or not bare or not yeto: + return "missing value" + if len(bare) < MIN_REPEATS or len(yeto) < MIN_REPEATS: + return f"insufficient repeats (bare {len(bare)}, yeto {len(yeto)}; need {MIN_REPEATS})" + if max(yeto) < min(bare): + return "yeto better" if lower_is_better else "yeto worse" + if min(yeto) > max(bare): + return "yeto worse" if lower_is_better else "yeto better" + return "within run-to-run range" + + +def _delta(bare: list[float], yeto: list[float]) -> str: + if not bare or not yeto or any(v is None for v in bare + yeto): + return "missing" + b, y = statistics.median(bare), statistics.median(yeto) + if b == 0: + return f"{y - b:+.4g} (abs)" + return f"{y - b:+.4g} ({(y - b) / abs(b) * 100:+.1f} %)" + + +def markdown(parsed_runs: list[dict[str, Any]], *, title: str) -> str: + if not parsed_runs: + return f"# {title}\n\nNo parsed runs.\n" + prints = {p["fingerprint"] for p in parsed_runs} + if len(prints) > 1: + raise ValueError(f"runs come from different specs (fingerprints {sorted(prints)}); report them separately") + spec = parsed_runs[0]["spec"] + summaries = [summarize(p) for p in parsed_runs] + by_arm: dict[str, list[dict]] = defaultdict(list) + for s in summaries: + by_arm[s["arm"]].append(s) + lines = [f"# {title}", "", + f"Spec `{spec['name']}`, fingerprint `{parsed_runs[0]['fingerprint'][:16]}`: model `{spec['model']}`@" + f"`{spec['model_revision'][:10]}`, data `{spec['data']}`@`{spec['data_revision'][:10]}`, " + f"{spec['rounds']} rounds x {spec['groups_per_round']} prompts x {spec['samples_per_group']} samples, " + f"max_tokens {spec['max_response_len']}, lr {spec['lr']}, LoRA r {spec['lora_r']}, seed {spec['seed']}, " + f"{spec['gpu']} (sync 1 card colocated, async 1 rollout + 1 trainer card), " + f"max_policy_age {spec['max_policy_age']} (async), TIS upper {spec['tis_upper']}.", "", + "Steady window = rounds after the first. Throughput counts every card of the arm. " + "Costs are list-price estimates (`prices.json`), not a bill.", ""] + lines += _runs_table(summaries) + _arms_table(by_arm) + _overhead(by_arm) + _yeto_segments(by_arm) + lines += _knobs(spec) + _rewards(summaries) + _missing(by_arm) + return "\n".join(lines) + "\n" + + +def _runs_table(summaries: list[dict]) -> list[str]: + head = ("arm", "run", "rc", "rounds", "steady wall s", "gen tok/card/s", "train tok/card/s", "round s med", + "util %", "bubble", "mem peak MiB", "$/1k steady", "$/1k run", "reward last3") + rows = ["## Runs", "", "| " + " | ".join(head) + " |", "|" + "---|" * len(head)] + for s in sorted(summaries, key=lambda s: (ARMS.index(s["arm"]), s["run_id"])): + vals = (s["arm"], f"`{s['run_id']}`", s["rc"], f"{s['rounds']}/{s['rounds_expected']}", s["steady_wall_s"], + s["gen_tok_per_card_s"], s["train_tok_per_card_s"], s["round_s_median"], s["gpu_util_mean"], + s["bubble_frac"], s["gpu_mem_peak_mb"], s["usd_per_1k_samples_steady"], s["usd_per_1k_samples_run"], + s["reward_last3_mean"]) + rows.append("| " + " | ".join(_fmt(v) for v in vals) + " |") + return rows + [""] + + +def _arms_table(by_arm: dict[str, list[dict]]) -> list[str]: + rows = ["## Per arm (median [min, max] over repeats)", "", + "| metric | " + " | ".join(a for a in ARMS if a in by_arm) + " |", "|---|" + "---|" * len(by_arm)] + for key, (label, _) in COMPARED.items(): + cells = [] + for a in ARMS: + if a in by_arm: + vals = [s[key] for s in by_arm[a]] + cells.append(_range([v for v in vals if v is not None]) if any(v is not None for v in vals) + else ("n/a" if all(s["sources"].get(key.removesuffix("_median")) == "n/a" + for s in by_arm[a]) else "missing")) + rows.append(f"| {label} | " + " | ".join(cells) + " |") + return rows + [""] + + +def _overhead(by_arm: dict[str, list[dict]]) -> list[str]: + rows = ["## yeto vs bare (same framework and mode)", "", + "Delta = median(yeto) - median(bare); positive time = yeto slower.", ""] + for fw in FRAMEWORKS: + for mode in MODES: + bare, yeto = by_arm.get(f"bare-{fw}-{mode}", []), by_arm.get(f"yeto-{fw}-{mode}", []) + if not bare and not yeto: + continue + rows += [f"### {fw} {mode} (bare n={len(bare)}, yeto n={len(yeto)})", "", + "| metric | bare | yeto | delta | verdict |", "|---|---|---|---|---|"] + for key, (label, lower) in COMPARED.items(): + bv, yv = [s[key] for s in bare], [s[key] for s in yeto] + if all(s["sources"].get(key.removesuffix("_median")) == "n/a" for s in bare + yeto): + rows.append(f"| {label} | n/a | n/a | n/a | not applicable |") + continue + rows.append(f"| {label} | {_range([v for v in bv if v is not None])} | " + f"{_range([v for v in yv if v is not None])} | {_delta(bv, yv)} | " + f"{verdict(bv, yv, lower)} |") + rows.append("") + return rows + + +def _yeto_segments(by_arm: dict[str, list[dict]]) -> list[str]: + rows = ["## Native segments per round (median, steady window)", "", + "yeto: tape task/phase names; bare: the framework's own timers. Where yeto adds work " + "(offload/onload, outer sync, publish), it shows here.", ""] + for a in ARMS: + for s in by_arm.get(a, []): + raw = s.get("segments_raw_median") or {} + rows.append(f"- {a} `{s['run_id']}`: " + (", ".join(f"{k} {_fmt(v)}" for k, v in raw.items()) + or "missing")) + return rows + [""] + + +def _knobs(spec: dict) -> list[str]: + rows = ["## verl overrides that differ between bare and yeto (same spec)", ""] + try: + for mode in MODES: + diff = verl_knob_diff(EffSpec(**spec), mode) + rows.append(f"- {mode}: " + ("; ".join(f"`{k}` bare={v['bare']} yeto={v['yeto']}" + for k, v in diff.items()) or "none")) + except ImportError as exc: + rows.append(f"- not computed ({exc})") + rows += ["- Miles: the yeto launcher builds the Miles argv at run time; compare the launch log's argv with " + "the bare script (`arm.json`) by hand (not automated in E0.1).", ""] + return rows + + +def _rewards(summaries: list[dict]) -> list[str]: + rows = ["## Reward against wall time and samples", "", "Per run: round: t_s / samples / reward_mean.", ""] + for s in sorted(summaries, key=lambda s: (ARMS.index(s["arm"]), s["run_id"])): + pts = " ".join(f"{c['round']}:{_fmt(c['t_s'])}/{_fmt(c['samples'])}/{_fmt(c['reward_mean'])}" + for c in s["reward_curve"]) + rows.append(f"- {s['arm']} `{s['run_id']}`: {pts or 'missing'}") + return rows + [""] + + +def _missing(by_arm: dict[str, list[dict]]) -> list[str]: + rows = ["## Field sources and gaps", "", "Per arm (first run): each round field's source; `missing` = no " + "source in the raw data, `n/a` = does not apply.", ""] + for a in ARMS: + if a in by_arm: + src = by_arm[a][0]["sources"] + rows.append(f"- {a}: " + "; ".join(f"{k}: {v}" for k, v in src.items())) + return rows + [""] diff --git a/tools/eval/efficiency/spec.py b/tools/eval/efficiency/spec.py new file mode 100644 index 00000000..be0e6a0a --- /dev/null +++ b/tools/eval/efficiency/spec.py @@ -0,0 +1,451 @@ +"""One shared spec per model preset, and the command plan of every arm. + +An arm is ``--``: + +* system ``bare`` (the framework's own entry point, no yeto code on its path) + or ``yeto`` (the same framework driven by yeto); +* framework ``verl`` (fork pin ``VERL_COMMIT``, the verl image yeto builds) or + ``miles`` (``MILES_NEXT_IMAGE``, the image yeto runs Miles in); +* mode ``sync`` (one card, rollout and trainer colocated) or ``async`` + (two cards, one rollout + one trainer, fully_async with a policy-age limit). + +Every value the comparison depends on comes from :class:`EffSpec`; +:func:`control_fingerprint` hashes them and every arm records it. + +Sources of the bare command lines (the "official recipe" each one follows): + +* bare verl sync: ``examples/grpo_trainer/run_qwen3_4b_fsdp.sh`` (fork + acad9875), ``python -m verl.trainer.main_ppo`` with its override groups; +* bare verl async: ``verl/experimental/fully_async_policy/shell/ + dapo_7b_math_fsdp2_4_4.sh``, ``python -m verl.experimental.fully_async_policy.fully_async_main``; +* bare Miles sync: ``scripts/run_qwen3_4b.py`` (Megatron branch, bridge mode) + -> ``train.py``; model flags from the image's own ``scripts/models/``; +* bare Miles async: ``examples/infra_features/fully_async/`` (``--fully-async``) + -> ``train_async.py``. + +Values the spec does not name stay at the recipe's value, or at the +framework default where the recipe sets none. Where yeto sets a value the +recipe also sets (LoRA, TIS, dtype), the bare arm takes the spec value, so the +two arms differ only by what yeto adds; :func:`verl_knob_diff` lists the verl +overrides that still differ. +""" + +from __future__ import annotations + +import hashlib +import json +import shlex +from dataclasses import asdict, dataclass, field, replace +from pathlib import Path +from typing import Any + +SYSTEMS = ("bare", "yeto") +FRAMEWORKS = ("verl", "miles") +MODES = ("sync", "async") +ARMS = tuple(f"{s}-{f}-{m}" for f in FRAMEWORKS for m in MODES for s in SYSTEMS) + +EFF_DIR = "/root/eff" # spec, reward and prep script inside a bare container +OUT_DIR = "/root/eff-out" # everything a container writes for the results Volume +DATA_DIR = "/root/eff-data" # eff_prep.py output +MODEL_DIR = "/root/eff-models" +VERL_ROOT = "/workspace/verl" +MILES_ROOT = "/root/miles" +MEGATRON_ROOT = "/root/Megatron-LM" # yeto.launch.remote_setup.PORTS_MEGATRON_PATH + + +@dataclass(frozen=True) +class EffSpec: + """Everything the arms of one comparison share.""" + + name: str + model: str + model_revision: str + miles_model_args: str # file under /scripts/models/ (the official model flags) + data: str = "zhuzilin/gsm8k" + data_revision: str = "0cbd9f31d91ac21a7613dcbc7fef992adac459ae" + gpu: str = "A10G" # Modal GPU type, the same for every arm + rounds: int = 8 # optimizer steps (= policy versions) per run + groups_per_round: int = 8 # prompts per step + samples_per_group: int = 4 # responses per prompt + max_prompt_len: int = 512 + max_response_len: int = 256 # "max_tokens" of the rollout + lr: float = 1e-5 + lora_r: int = 32 # yeto's verl backend is LoRA only, so every arm is LoRA + seed: int = 17 + temperature: float = 1.0 + top_p: float = 1.0 + mem_fraction: float = 0.5 # rollout engine static memory fraction + max_policy_age: int = 1 # async arms: versions a sample may lag + tis_upper: float = 2.0 # token-level truncated IS upper bound (all arms) + repeats: int = 2 # runs per arm (range verdicts need >= 2) + timeout_s: int = 3000 + chat_template_kwargs: dict[str, Any] = field(default_factory=dict) + + @property + def samples_per_round(self) -> int: + return self.groups_per_round * self.samples_per_group + + @property + def seq_len(self) -> int: + return self.max_prompt_len + self.max_response_len + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +PRESETS: dict[str, EffSpec] = { + # E0.1 acceptance (debug size): the T-EVAL model and data on the T-EVAL card + "qwen3-0.6b": EffSpec( + name="qwen3-0.6b", model="Qwen/Qwen3-0.6B", model_revision="c1899de289a04d12100db370d81485cdf75e47ca", + miles_model_args="qwen3-0.6B.py", chat_template_kwargs={"enable_thinking": False}), + # E0.2 (main comparison): revision read from the HF API 2026-10-20 + "qwen3-4b-instruct-2507": EffSpec( + name="qwen3-4b-instruct-2507", model="Qwen/Qwen3-4B-Instruct-2507", + model_revision="cdbee75f17c01a7cc42f958dc650907174af0554", miles_model_args="qwen3-4B-Instruct-2507.py", + gpu="H100!", rounds=20, groups_per_round=32, samples_per_group=8, max_prompt_len=512, + max_response_len=1024, lora_r=64, mem_fraction=0.6, timeout_s=5400), +} + + +@dataclass(frozen=True) +class Arm: + system: str + framework: str + mode: str + + @classmethod + def parse(cls, name: str) -> "Arm": + parts = name.split("-") + if len(parts) != 3 or name not in ARMS: + raise ValueError(f"unknown arm {name!r}; known: {', '.join(ARMS)}") + return cls(*parts) + + @property + def name(self) -> str: + return f"{self.system}-{self.framework}-{self.mode}" + + @property + def peer(self) -> str: + """The arm this one is compared with (same framework and mode).""" + return f"{'yeto' if self.system == 'bare' else 'bare'}-{self.framework}-{self.mode}" + + +def get_spec(preset: str, **overrides: Any) -> EffSpec: + if preset not in PRESETS: + raise ValueError(f"unknown preset {preset!r}; known: {', '.join(PRESETS)}") + return replace(PRESETS[preset], **overrides) if overrides else PRESETS[preset] + + +def cards(mode: str) -> dict[str, int]: + """Cards per role: sync colocates both on one card; async has one of each.""" + return {"total": 1, "rollout": 1, "trainer": 1, "colocated": True} if mode == "sync" else \ + {"total": 2, "rollout": 1, "trainer": 1, "colocated": False} + + +def control_fingerprint(spec: EffSpec) -> str: + """sha256 over every shared value except the repeat count (a report refuses + to compare runs whose fingerprints differ).""" + doc = {k: v for k, v in spec.to_dict().items() if k not in ("repeats", "timeout_s")} + return hashlib.sha256(json.dumps(doc, sort_keys=True).encode()).hexdigest() + + +def images() -> dict[str, str]: + """The image pins every arm of a framework uses (same image for bare and yeto).""" + from yeto.rl.adapters.miles.pins import MILES_NEXT_IMAGE + from yeto.rl.adapters.verl.pins import VERL_COMMIT + + return {"verl": f"verl-build:{VERL_COMMIT}", "miles": MILES_NEXT_IMAGE} + + +def _model_path(spec: EffSpec) -> str: + return f"{MODEL_DIR}/{spec.model.split('/')[-1]}" + + +# ====================================================================== bare verl +def verl_bare_overrides(spec: EffSpec, mode: str, run_id: str) -> list[str]: + """Hydra overrides of the bare verl arm (recipe groups in the recipe's order).""" + c = cards(mode) + data = [ + "algorithm.adv_estimator=grpo", + "algorithm.use_kl_in_reward=False", + "algorithm.norm_adv_by_std_in_grpo=True", + f"data.train_files={DATA_DIR}/train.parquet", + f"data.val_files={DATA_DIR}/test.parquet", + f"data.max_prompt_length={spec.max_prompt_len}", + f"data.max_response_length={spec.max_response_len}", + "data.filter_overlong_prompts=True", + "data.truncation=error", + f"data.seed={spec.seed}", + *[f"+data.apply_chat_template_kwargs.{k}={v}" for k, v in sorted(spec.chat_template_kwargs.items())], + ] + model = [ + f"actor_rollout_ref.model.path={_model_path(spec)}", + "actor_rollout_ref.model.use_remove_padding=True", + "actor_rollout_ref.model.enable_gradient_checkpointing=True", + f"actor_rollout_ref.model.lora_rank={spec.lora_r}", + f"actor_rollout_ref.model.lora_alpha={spec.lora_r}", + "actor_rollout_ref.model.target_modules=all-linear", + ] + actor = [ + "actor_rollout_ref.actor.strategy=fsdp2", + "actor_rollout_ref.actor.fsdp_config.model_dtype=fp32", # yeto's VerlRunConfig default + f"actor_rollout_ref.actor.optim.lr={spec.lr!r}", + f"actor_rollout_ref.actor.ppo_mini_batch_size={spec.groups_per_round}", + "actor_rollout_ref.actor.use_dynamic_bsz=True", + "actor_rollout_ref.actor.ppo_max_token_len_per_gpu=16384", + "actor_rollout_ref.actor.use_kl_loss=False", + "actor_rollout_ref.actor.entropy_coeff=0", + "actor_rollout_ref.actor.fsdp_config.param_offload=False", + "actor_rollout_ref.actor.fsdp_config.optimizer_offload=False", + f"actor_rollout_ref.actor.data_loader_seed={spec.seed}", + ] + rollout = [ + "actor_rollout_ref.rollout.name=vllm", + "actor_rollout_ref.rollout.tensor_model_parallel_size=1", + f"actor_rollout_ref.rollout.gpu_memory_utilization={spec.mem_fraction}", + f"actor_rollout_ref.rollout.n={spec.samples_per_group}", + f"actor_rollout_ref.rollout.temperature={spec.temperature}", + f"actor_rollout_ref.rollout.top_p={spec.top_p}", + "actor_rollout_ref.rollout.top_k=-1", + "actor_rollout_ref.rollout.calculate_log_probs=True", + "actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True", + "actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=16384", + "algorithm.rollout_correction.rollout_is=token", + f"algorithm.rollout_correction.rollout_is_threshold={spec.tis_upper}", + "algorithm.rollout_correction.bypass_mode=False", + ] + reward = [f"reward.custom_reward_function.path={EFF_DIR}/eff_reward.py", + "reward.custom_reward_function.name=compute_score"] + trainer = [ + "trainer.critic_warmup=0", + "trainer.logger=[console,file]", + "trainer.project_name=yeto-effeval", + f"trainer.experiment_name={run_id}", + f"trainer.default_local_dir={OUT_DIR}/ckpt", + "trainer.nnodes=1", + "trainer.val_before_train=False", + "trainer.test_freq=-1", + "trainer.save_freq=-1", + "trainer.resume_mode=disable", + f"trainer.total_training_steps={spec.rounds}", + "trainer.total_epochs=1000", + "ray_kwargs.ray_init.runtime_env.py_executable=null", + "ray_kwargs.ray_init.num_cpus=32", # a Modal container reports too few Ray CPUs (S17 V2) + ] + if mode == "sync": + return [*data, f"data.train_batch_size={spec.groups_per_round}", *model, *actor, *rollout, *reward, + *trainer, f"trainer.n_gpus_per_node={c['total']}"] + fully_async = [ # dapo_7b_math_fsdp2_4_4.sh; one parameter version per spec round + "data.train_batch_size=0", + "data.gen_batch_size=1", + "data.return_raw_chat=True", + "actor_rollout_ref.hybrid_engine=False", + "actor_rollout_ref.rollout.mode=async", + "actor_rollout_ref.rollout.checkpoint_engine.backend=nccl", + "rollout.nnodes=1", + f"rollout.n_gpus_per_node={c['rollout']}", + f"rollout.total_rollout_steps={spec.groups_per_round * spec.rounds}", + f"trainer.n_gpus_per_node={c['trainer']}", + f"async_training.staleness_threshold={spec.max_policy_age}", + "async_training.trigger_parameter_sync_step=1", + "async_training.require_batches=1", + "async_training.partial_rollout=True", + "async_training.use_trainer_do_validate=False", + "async_training.use_dynamic_resource_scheduling=False", + ] + return [*data, *model, *actor, *rollout, *reward, *trainer, *fully_async] + + +def verl_bare_entry(mode: str) -> str: + return "verl.trainer.main_ppo" if mode == "sync" else "verl.experimental.fully_async_policy.fully_async_main" + + +# ====================================================================== bare Miles +def miles_bare_argv(spec: EffSpec, mode: str) -> list[str]: + """Miles arguments after the model flags (the container prepends ``$MODEL_ARGS``).""" + c = cards(mode) + model = _model_path(spec) + argv = [ + # checkpoint: bridge mode loads the HF checkpoint directly (run_qwen3_4b.py --megatron-to-hf-mode bridge) + "--hf-checkpoint", model, "--ref-load", model, "--megatron-to-hf-mode", "bridge", + "--finetune", "--no-load-optim", "--no-load-rng", "--no-save-optim", "--no-save-rng", + # rollout + "--prompt-data", f"{DATA_DIR}/train.jsonl", "--input-key", "prompt", "--label-key", "label", + "--apply-chat-template", "--rollout-shuffle", "--rollout-seed", str(spec.seed), + "--custom-rm-path", "eff_reward.score", + "--num-rollout", str(spec.rounds), "--rollout-batch-size", str(spec.groups_per_round), + "--n-samples-per-prompt", str(spec.samples_per_group), + "--rollout-max-response-len", str(spec.max_response_len), + "--rollout-max-context-len", str(spec.seq_len), + "--rollout-temperature", str(spec.temperature), "--rollout-top-p", str(spec.top_p), + "--global-batch-size", str(spec.samples_per_round), "--balance-data", + # GRPO + TIS (the correction every arm uses) + "--advantage-estimator", "grpo", "--entropy-coef", "0.00", "--eps-clip", "0.2", "--eps-clip-high", "0.28", + "--use-tis", "--tis-clip", str(spec.tis_upper), + # optimizer + "--optimizer", "adam", "--lr", str(spec.lr), "--lr-decay-style", "constant", + "--seed", str(spec.seed), + # LoRA (Miles' own flags; the same set yeto's Miles adapter emits) + "--lora-rank", str(spec.lora_r), "--lora-alpha", str(spec.lora_r), "--lora-dropout", "0", + "--lora-type", "canonical_lora", "--target-modules", "all-linear", + "--sglang-max-lora-rank", str(spec.lora_r), + # Megatron (run_qwen3_4b.py, Megatron branch) + "--train-backend", "megatron", "--seq-length", str(spec.seq_len), + "--tensor-model-parallel-size", "1", "--pipeline-model-parallel-size", "1", + "--context-parallel-size", "1", "--expert-model-parallel-size", "1", "--expert-tensor-parallel-size", "1", + "--recompute-granularity", "full", "--recompute-method", "uniform", "--recompute-num-layers", "1", + "--use-dynamic-batch-size", "--max-tokens-per-gpu", str(max(9216, spec.seq_len)), + "--attention-dropout", "0.0", "--hidden-dropout", "0.0", + "--accumulate-allreduce-grads-in-fp32", "--attention-softmax-in-fp32", "--attention-backend", "flash", + "--bf16", + # SGLang + "--rollout-num-gpus-per-engine", "1", "--sglang-mem-fraction-static", str(spec.mem_fraction), + # placement + "--actor-num-nodes", "1", "--actor-num-gpus-per-node", str(c["trainer"]), + "--num-gpus-per-node", str(c["total"]), + ] + if spec.chat_template_kwargs: + argv += ["--apply-chat-template-kwargs", json.dumps(spec.chat_template_kwargs, separators=(",", ":"))] + if mode == "sync": + argv += ["--colocate", "--offload-train", "--lora-base-cpu-backup"] + else: + argv += ["--rollout-num-gpus", str(c["rollout"]), "--fully-async", + "--max-weight-staleness", str(spec.max_policy_age)] + return argv + + +def miles_bare_script(mode: str) -> str: + return "train.py" if mode == "sync" else "train_async.py" + + +# ====================================================================== yeto arms +def yeto_verl_argv(spec: EffSpec, mode: str, tape: str) -> list[str]: + """``island_entry`` arguments of the yeto+verl arm (the T-EVAL E1 shape).""" + argv = ["-m", "yeto.rl.adapters.verl.island_entry", "--model", spec.model, "--model-revision", + spec.model_revision, "--data", spec.data, "--data-revision", spec.data_revision, "--learner-id", "0", + "--reward-function", "gsm8k_reward:score", "--global-rounds", str(spec.rounds), + "--groups-per-round", str(spec.groups_per_round), "--samples-per-group", str(spec.samples_per_group), + "--rollout-max-response-len", str(spec.max_response_len), "--event-tape", tape, + "--inner-lr", str(spec.lr), "--lora-r", str(spec.lora_r), "--lora-targets", "all-linear", + "--seq-len", str(spec.seq_len), "--seed", str(spec.seed), "--rl-single-island-no-sync"] + if spec.chat_template_kwargs: + argv += ["--apply-chat-template-kwargs", json.dumps(spec.chat_template_kwargs)] + if mode == "async": + argv += ["--rl-max-policy-age", str(spec.max_policy_age)] + return argv + + +def yeto_miles_launcher_args(spec: EffSpec, mode: str, run_id: str) -> str: + """The yeto launcher arguments of the yeto+Miles arm (``tools/eval/miles_local.sh`` ARGS).""" + from ..miles import DATA, SPEC, STORE_VOLUME, TAPE_VOLUME + + card = {"A10G": "a10g", "H100!": "h100", "L4": "l4"}[spec.gpu] + c = cards(mode) + kwargs = json.dumps(spec.chat_template_kwargs) if spec.chat_template_kwargs else "" + args = (f"launch --training-mode rl --rl-engine ports --preflight-threads off --modal-gpu-exact " + f"--modal-retries 0 --modal-cpu 8 --modal-memory 64 --modal-tape-volume {TAPE_VOLUME} " + f"--controller local --cluster-prefix {run_id} --tuning lora --lora-r {spec.lora_r} " + f"--rl-observe-timeline --model {spec.model} --model-revision {spec.model_revision} {DATA} " + f"--rl-checkpoint-store modal-volume://{STORE_VOLUME}/{run_id} " + "--reward-function gsm8k_reward:score --fragments 1 --pipeline 1 " + f"--rollout-batch-size {spec.groups_per_round} --n-samples-per-prompt {spec.samples_per_group} " + f"--rollout-max-response-len {spec.max_response_len} --seq-len {spec.seq_len} " + f"--inner-lr {spec.lr} --seed {spec.seed} --sglang-mem-fraction-static {spec.mem_fraction} " + f"--gpu modal:{c['total']}x{card} --rl-single-island-no-sync --total-steps {spec.rounds} " + # TIS (upper 2.0) in both modes, as in the bare arms; tolerates staleness 2 >= max_policy_age + f"--trust-remote-code {SPEC}") + if kwargs: + args += f" --apply-chat-template-kwargs {shlex.quote(kwargs)}" + if mode == "async": + args += (f" --rl-placement fixed-partition --rl-rollout-gpus {c['rollout']} " + f"--rl-max-policy-age {spec.max_policy_age}") + if spec.data != "zhuzilin/gsm8k" or spec.data_revision not in DATA: + raise ValueError("the yeto+Miles arm reads tools.eval.miles.DATA; keep the spec's data equal to it") + return args + + +# ====================================================================== plans +def bare_script(spec: EffSpec, arm: Arm, run_id: str) -> str: + """The bash script a bare container runs (after eff_prep.py staged model and data).""" + c = cards(arm.mode) + lines = ["set -euo pipefail", f"mkdir -p {OUT_DIR}", + f"python3 {EFF_DIR}/eff_prep.py {EFF_DIR}/spec.json > {OUT_DIR}/prep.json"] + if arm.framework == "verl": + lines += [f"cd {VERL_ROOT}", f"export VERL_FILE_LOGGER_PATH={OUT_DIR}/verl-metrics.jsonl", + f"export PYTHONPATH={EFF_DIR}", + "exec python3 -m " + verl_bare_entry(arm.mode) + " " + shlex.join( + verl_bare_overrides(spec, arm.mode, run_id))] + else: + load = ("import importlib.util as u, sys; sys.path.insert(0, '{d}'); " + "s = u.spec_from_file_location('m', '{d}/{f}'); m = u.module_from_spec(s); " + "s.loader.exec_module(m); print(m.model_args())").format(d=f"{MILES_ROOT}/scripts/models", + f=spec.miles_model_args) + runtime_env = json.dumps({"env_vars": {"PYTHONPATH": f"{MEGATRON_ROOT}/:{EFF_DIR}", + "CUDA_DEVICE_MAX_CONNECTIONS": "1"}}) + lines += [f"cd {MILES_ROOT}", f"MODEL_ARGS=$(python3 -c {shlex.quote(load)})", + f"echo \"$MODEL_ARGS\" > {OUT_DIR}/model_args.txt", + f"ray start --head --node-ip-address 127.0.0.1 --num-gpus {c['total']} --num-cpus 32 " + "--disable-usage-stats --dashboard-host=127.0.0.1", + f"exec ray job submit --address=http://127.0.0.1:8265 --runtime-env-json={shlex.quote(runtime_env)} " + f"-- python3 {MILES_ROOT}/{miles_bare_script(arm.mode)} $MODEL_ARGS " + + shlex.join(miles_bare_argv(spec, arm.mode))] + return "\n".join(lines) + "\n" + + +def plan(spec: EffSpec, arm_name: str, run_id: str) -> dict[str, Any]: + """Everything one run of an arm needs (written to ``arm.json`` before the launch).""" + arm = Arm.parse(arm_name) + c = cards(arm.mode) + doc: dict[str, Any] = { + "arm": arm.name, "system": arm.system, "framework": arm.framework, "mode": arm.mode, "run_id": run_id, + "spec": spec.to_dict(), "fingerprint": control_fingerprint(spec), "cards": c, "gpu": spec.gpu, + } + if arm.system == "bare": + doc.update(harness="modal_eff", image=arm.framework, script=bare_script(spec, arm, run_id)) + elif arm.framework == "verl": + doc.update(harness="modal_eff", image="verl", + argv=yeto_verl_argv(spec, arm.mode, "/root/yeto-output/rl-island-0.jsonl")) + else: + doc.update(harness="miles_local", launcher_args=yeto_miles_launcher_args(spec, arm.mode, run_id)) + return doc + + +# ====================================================================== control check +def _override_map(overrides: list[str]) -> dict[str, str]: + out = {} + for o in overrides: + key, _, value = o.partition("=") + out[key.lstrip("+")] = value + return out + + +def verl_knob_diff(spec: EffSpec, mode: str) -> dict[str, dict[str, str | None]]: + """Hydra overrides where bare verl and yeto+verl differ (key -> {bare, yeto}). + + The yeto side is yeto's own translation (``build_overrides``, plus + ``fully_async_run_overrides`` for async) of the same spec; paths, names and + keys yeto handles in its own loop are compared too, so the list is the full + set of differences, not only the interesting ones.""" + from yeto.rl.adapters.verl.config import VerlRunConfig, build_overrides + from yeto.rl.adapters.verl.fully_async_round import fully_async_run_overrides + + cfg = VerlRunConfig( + model_path=_model_path(spec), train_file=f"{DATA_DIR}/train.parquet", val_file=f"{DATA_DIR}/test.parquet", + out_dir=OUT_DIR, lora_rank=spec.lora_r, lora_alpha=spec.lora_r, lr=spec.lr, + groups_per_round=spec.groups_per_round, samples_per_group=spec.samples_per_group, + max_prompt_len=spec.max_prompt_len, max_response_len=spec.max_response_len, seed=spec.seed, + total_rounds=spec.rounds, temperature=spec.temperature, top_p=spec.top_p, tis_upper=spec.tis_upper, + gpu_memory_utilization=spec.mem_fraction, reward_path=f"{EFF_DIR}/eff_reward.py", + chat_template_kwargs=dict(spec.chat_template_kwargs), experiment_name="x") + yeto = build_overrides(cfg) + if mode == "async": + yeto = fully_async_run_overrides(yeto, spec.max_policy_age, groups_per_round=spec.groups_per_round, + rounds=spec.rounds) + bare, ours = _override_map(verl_bare_overrides(spec, mode, "x")), _override_map(yeto) + return {k: {"bare": bare.get(k), "yeto": ours.get(k)} for k in sorted(set(bare) | set(ours)) + if bare.get(k) != ours.get(k)} + + +def write_plan(path: Path, doc: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(doc, indent=1, default=str)) From 261ba2e1695783aff28df5a220dd66dccc4a5465 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:52:50 +0000 Subject: [PATCH 02/12] tools/eval: bare Miles trains with unfused attention (match yeto generic recipe; TE flash fails on sm86) Co-Authored-By: Claude Opus 5.5 --- tools/eval/efficiency/spec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/eval/efficiency/spec.py b/tools/eval/efficiency/spec.py index be0e6a0a..ea0f79cb 100644 --- a/tools/eval/efficiency/spec.py +++ b/tools/eval/efficiency/spec.py @@ -296,7 +296,10 @@ def miles_bare_argv(spec: EffSpec, mode: str) -> list[str]: "--recompute-granularity", "full", "--recompute-method", "uniform", "--recompute-num-layers", "1", "--use-dynamic-batch-size", "--max-tokens-per-gpu", str(max(9216, spec.seq_len)), "--attention-dropout", "0.0", "--hidden-dropout", "0.0", - "--accumulate-allreduce-grads-in-fp32", "--attention-softmax-in-fp32", "--attention-backend", "flash", + "--accumulate-allreduce-grads-in-fp32", "--attention-softmax-in-fp32", + # yeto's generic recipe (Qwen3) trains with "unfused": TE's flash path picks the + # FlashAttention cute kernel, which fails to compile on A10G (sm86, eff01-bare-miles-sync-a). + "--attention-backend", "unfused", "--bf16", # SGLang "--rollout-num-gpus-per-engine", "1", "--sglang-mem-fraction-static", str(spec.mem_fraction), From c3c9d09d8184a2ea3c5b5e5dce12be005337bc49 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:20:51 +0000 Subject: [PATCH 03/12] tools/eval: yeto+verl async arm passes the staleness-2 TIS algorithm spec Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 4 ++++ tools/eval/efficiency/spec.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py index a7ff78f6..b1ca57df 100644 --- a/tests/test_tools_eval_efficiency.py +++ b/tests/test_tools_eval_efficiency.py @@ -78,6 +78,10 @@ def test_bare_entry_points_follow_the_official_recipes(): def test_yeto_arms_use_the_same_policy_age_and_partition(): assert "--rl-max-policy-age 1" in " ".join(spec.yeto_verl_argv(SPEC, "async", "/t")) assert "--rl-max-policy-age" not in " ".join(spec.yeto_verl_argv(SPEC, "sync", "/t")) + # async needs an algorithm spec that tolerates the policy age (the default allows 0) + argv = spec.yeto_verl_argv(SPEC, "async", "/t") + algo = argv[argv.index("--rl-algorithm-spec") + 1] + assert algo.endswith(spec.ALGORITHM_SPEC) and (Path(__file__).parents[1] / spec.ALGORITHM_SPEC).is_file() args = spec.yeto_miles_launcher_args(SPEC, "async", "r1") assert "--gpu modal:2xa10g" in args and "--rl-placement fixed-partition --rl-rollout-gpus 1" in args assert "--gpu modal:1xa10g" in spec.yeto_miles_launcher_args(SPEC, "sync", "r1") diff --git a/tools/eval/efficiency/spec.py b/tools/eval/efficiency/spec.py index ea0f79cb..b10acc48 100644 --- a/tools/eval/efficiency/spec.py +++ b/tools/eval/efficiency/spec.py @@ -322,6 +322,8 @@ def miles_bare_script(mode: str) -> str: # ====================================================================== yeto arms +YETO_WORKDIR = "/root/sky_workdir" # modal_verl.WORKDIR: the repo inside the yeto+verl container +ALGORITHM_SPEC = "examples/rl_algorithms/decoupled-ppo-c2.json" # token TIS 2.0, max_policy_staleness 2 def yeto_verl_argv(spec: EffSpec, mode: str, tape: str) -> list[str]: """``island_entry`` arguments of the yeto+verl arm (the T-EVAL E1 shape).""" argv = ["-m", "yeto.rl.adapters.verl.island_entry", "--model", spec.model, "--model-revision", @@ -334,7 +336,10 @@ def yeto_verl_argv(spec: EffSpec, mode: str, tape: str) -> list[str]: if spec.chat_template_kwargs: argv += ["--apply-chat-template-kwargs", json.dumps(spec.chat_template_kwargs)] if mode == "async": - argv += ["--rl-max-policy-age", str(spec.max_policy_age)] + # The default algorithm spec allows no staleness (island_entry rejects age 1, + # eff01-yeto-verl-async-a); use the yeto+Miles arm's spec: token TIS 2.0, staleness 2. + argv += ["--rl-max-policy-age", str(spec.max_policy_age), + "--rl-algorithm-spec", f"{YETO_WORKDIR}/{ALGORITHM_SPEC}"] return argv From 63dc064482fa86b0ae42a2518433150471a6e75d Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:02:58 +0000 Subject: [PATCH 04/12] tools/eval: bare verl arms run the unpatched fork; async uses merged LoRA and /dev/shm NCCL host buffers The verl image applies yeto's patch_verl, whose sender hook imports yeto, so the bare arms now build the image without it. The fork's disaggregated checkpoint engine drops the PEFT config of unmerged LoRA params; the bare async arm sends merged weights instead. On Modal the trainer allocated NCCL host buffers with cuMem while the receiver fell back to /dev/shm and failed to attach (NCCL_DEBUG=INFO run); NCCL_CUMEM_HOST_ENABLE=0 keeps both on /dev/shm. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 6 ++++++ tools/eval/efficiency/modal_eff.py | 7 +++++-- tools/eval/efficiency/spec.py | 8 ++++++++ yeto/rl/adapters/verl/image.py | 15 +++++++++------ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py index b1ca57df..44356fbf 100644 --- a/tests/test_tools_eval_efficiency.py +++ b/tests/test_tools_eval_efficiency.py @@ -340,3 +340,9 @@ def test_cli_plan_and_unregistered_run(tmp_path, monkeypatch, capsys): (tmp_path / "ledger.md").write_text("nothing registered\n") with pytest.raises(SystemExit, match="not pre-registered"): cli.main(["eff-run", "qwen3-0.6b", "--arm", "bare-verl-sync", "--run-id", "eff-x", "--out", str(tmp_path)]) + + +def test_bare_verl_async_uses_the_fork_working_lora_sync(): + script = spec.bare_script(SPEC, spec.Arm.parse("bare-verl-async"), "r") + assert "actor_rollout_ref.model.lora.merge=True" in script and "export NCCL_CUMEM_HOST_ENABLE=0" in script + assert "NCCL_CUMEM_HOST_ENABLE" not in spec.bare_script(SPEC, spec.Arm.parse("bare-verl-sync"), "r") diff --git a/tools/eval/efficiency/modal_eff.py b/tools/eval/efficiency/modal_eff.py index 0bcf5719..d67dd09e 100644 --- a/tools/eval/efficiency/modal_eff.py +++ b/tools/eval/efficiency/modal_eff.py @@ -2,7 +2,8 @@ arms and yeto+verl (yeto+Miles runs through the yeto launcher, ``miles_local.sh``). One GPU container per run, ``:``, in the framework's image (the -same image the yeto arm of that framework uses). The container: +image the yeto arm of that framework uses; for verl the bare arms skip yeto's +source patch). The container: 1. checks every visible GPU is the asked type (``modal_verl._gpu``); 2. samples NVML every second into ``nvml.csv`` (index, util, memory, power); @@ -111,7 +112,9 @@ def build_app(run: str, plan: dict): from yeto.rl.adapters.verl.image import modal_image if plan["image"] == "verl": - image = modal_image(modal).env({"PYTHONUNBUFFERED": "1"}) + # bare arms run the fork as shipped: yeto's patch replaces the LoRA weight-sync + # sender with an import of yeto (eff01-bare-verl-async-a/d) + image = modal_image(modal, patched=plan["system"] == "yeto").env({"PYTHONUNBUFFERED": "1"}) if plan["system"] == "yeto": image = image.add_local_dir(str(mv.REPO), mv.WORKDIR, copy=False, ignore=MOUNT_IGNORE) else: diff --git a/tools/eval/efficiency/spec.py b/tools/eval/efficiency/spec.py index b10acc48..5b158ae5 100644 --- a/tools/eval/efficiency/spec.py +++ b/tools/eval/efficiency/spec.py @@ -242,6 +242,10 @@ def verl_bare_overrides(spec: EffSpec, mode: str, run_id: str) -> list[str]: "actor_rollout_ref.hybrid_engine=False", "actor_rollout_ref.rollout.mode=async", "actor_rollout_ref.rollout.checkpoint_engine.backend=nccl", + # The fork's disaggregated sender drops the PEFT config of unmerged LoRA params + # (engine_workers.update_weights), so the rollout would load them as base + # weights; merged LoRA sends full weights, the fork's working LoRA path. + "actor_rollout_ref.model.lora.merge=True", "rollout.nnodes=1", f"rollout.n_gpus_per_node={c['rollout']}", f"rollout.total_rollout_steps={spec.groups_per_round * spec.rounds}", @@ -381,6 +385,10 @@ def bare_script(spec: EffSpec, arm: Arm, run_id: str) -> str: if arm.framework == "verl": lines += [f"cd {VERL_ROOT}", f"export VERL_FILE_LOGGER_PATH={OUT_DIR}/verl-metrics.jsonl", f"export PYTHONPATH={EFF_DIR}", + # Modal: the trainer allocates NCCL host buffers with cuMem, the + # rollout-side receiver falls back to /dev/shm and cannot attach + # (eff01-bare-verl-async-d, NCCL_DEBUG=INFO); both use /dev/shm. + *(["export NCCL_CUMEM_HOST_ENABLE=0"] if arm.mode == "async" else []), "exec python3 -m " + verl_bare_entry(arm.mode) + " " + shlex.join( verl_bare_overrides(spec, arm.mode, run_id))] else: diff --git a/yeto/rl/adapters/verl/image.py b/yeto/rl/adapters/verl/image.py index b096d1b1..29a52282 100644 --- a/yeto/rl/adapters/verl/image.py +++ b/yeto/rl/adapters/verl/image.py @@ -46,15 +46,18 @@ def image_env() -> dict[str, str]: } -def modal_image(modal, commit: str = VERL_COMMIT): - patch = Path(__file__).with_name("patch_verl.py") - return ( +def modal_image(modal, commit: str = VERL_COMMIT, patched: bool = True): + """The verl image; ``patched=False`` leaves the verl source as the fork ships it + (the bare-verl arms of the efficiency evaluation, which must not import yeto).""" + image = ( modal.Image.from_registry(VERL_BASE_IMAGE, add_python=VERL_PYTHON) .apt_install("git", "build-essential", "curl", "libnuma1") .pip_install(f"uv=={VERL_UV_VERSION}") .run_commands(*build_commands(commit)) .run_commands(*store_client_commands()) - .add_local_file(str(patch), "/workspace/patch_verl.py", copy=True) - .run_commands(f"{VENV_BIN}/python /workspace/patch_verl.py {VERL_ROOT}") - .env(image_env()) ) + if patched: + patch = Path(__file__).with_name("patch_verl.py") + image = (image.add_local_file(str(patch), "/workspace/patch_verl.py", copy=True) + .run_commands(f"{VENV_BIN}/python /workspace/patch_verl.py {VERL_ROOT}")) + return image.env(image_env()) From 052c31e167f946f323dd4476521dc8cfa923ad66 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:36:04 +0000 Subject: [PATCH 05/12] tools/eval: parse bare verl fully_async rows by training step; rates over rounds that report the value The fork logs a step's training metrics one parameter version late and never logs the last step's, so rows are keyed by training/global_step and the step without metrics keeps its trained batch from the config. Throughput rates use the steady rounds that report the value; the report lists partial counts. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 18 ++++++++++++++++++ tools/eval/efficiency/metrics.py | 11 ++++++++--- tools/eval/efficiency/parse.py | 26 ++++++++++++++++++++------ tools/eval/efficiency/report.py | 4 ++++ 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py index 44356fbf..cdd9df04 100644 --- a/tests/test_tools_eval_efficiency.py +++ b/tests/test_tools_eval_efficiency.py @@ -346,3 +346,21 @@ def test_bare_verl_async_uses_the_fork_working_lora_sync(): script = spec.bare_script(SPEC, spec.Arm.parse("bare-verl-async"), "r") assert "actor_rollout_ref.model.lora.merge=True" in script and "export NCCL_CUMEM_HOST_ENABLE=0" in script assert "NCCL_CUMEM_HOST_ENABLE" not in spec.bare_script(SPEC, spec.Arm.parse("bare-verl-sync"), "r") + + +def test_parse_bare_verl_async_rows_lag_one_version(): + # fork acad9875 fully_async: weight-sync row at version N, training row of step N at N+1, + # the last step's training row never written + rows = [{"step": 0, "data": {"fully_async/rollouter/active_time": 9.0}}] + for v in range(1, 4): + rows.append({"step": v, "data": {"fully_async/rollouter/active_time": 1.0}}) + if v > 1: + rows.append({"step": v, "data": {"training/global_step": v - 1, "response_length/mean": 10.0, + "critic/score/mean": 0.5}}) + log = "\n".join(f"{100 + v}.0 (FullyAsyncTrainer) step:{v} - x:1" for v in range(4)) + p = parse.parse_verl("\n".join(json.dumps(r) for r in rows), log, samples_per_round=4, trainer_cards=1, + mode="async") + assert [r["round"] for r in p.rounds] == [1, 2, 3] + assert [r["t_end_unix"] for r in p.rounds] == [101.0, 102.0, 103.0] + assert [r["reward_mean"] for r in p.rounds] == [0.5, 0.5, None] + assert [r["samples"] for r in p.rounds] == [4, 4, 4] diff --git a/tools/eval/efficiency/metrics.py b/tools/eval/efficiency/metrics.py index 6fe298c0..0df3bfa2 100644 --- a/tools/eval/efficiency/metrics.py +++ b/tools/eval/efficiency/metrics.py @@ -58,11 +58,16 @@ def summarize(parsed: dict[str, Any], *, prices_path: Path = PRICES) -> dict[str "rc": parsed.get("rc")} wall = _sum_all(steady, "round_s") s["steady_wall_s"] = round(wall, 3) if wall is not None else None + s["rate_rounds"] = {} for key, out in (("gen_tokens", "gen_tok_per_card_s"), ("train_tokens", "train_tok_per_card_s"), ("samples", "samples_per_s")): - total = _sum_all(steady, key) - denom = (wall * (cards if key != "samples" else 1)) if wall else None - s[out] = round(total / denom, 3) if total is not None and denom else None + # over the steady rounds that report the value (bare verl fully_async never + # logs its last step's batch); the round count is kept when it is fewer + known = [r for r in steady if r.get(key) is not None and r.get("round_s") is not None] + time_s = sum(r["round_s"] for r in known) * (cards if key != "samples" else 1) + s[out] = round(sum(r[key] for r in known) / time_s, 3) if known and time_s else None + if known and len(known) < len(steady): + s["rate_rounds"][out] = f"{len(known)}/{len(steady)}" s["round_s_median"] = _median(_vals(steady, "round_s")) s["round_s_range"] = [min(_vals(steady, "round_s")), max(_vals(steady, "round_s"))] if _vals(steady, "round_s") \ else None diff --git a/tools/eval/efficiency/parse.py b/tools/eval/efficiency/parse.py index 167d0bab..066f8d6f 100644 --- a/tools/eval/efficiency/parse.py +++ b/tools/eval/efficiency/parse.py @@ -128,13 +128,24 @@ def parse_verl(metrics_jsonl: str, log_text: str = "", *, samples_per_round: int """verl FileLogger rows (+ console step lines for end times).""" p = Parsed() ends = _step_ends(log_text, VERL_STEP_LINE) + recs: dict[int, dict[str, Any]] = {} + segs: dict[int, dict[str, float]] = {} for line in metrics_jsonl.splitlines(): if not line.strip(): continue row = json.loads(line) d, step = row.get("data") or {}, int(row["step"]) - rec = _record(step) - _set(p, rec, "t_end_unix", ends.get(step), "console step line (harness time prefix)") + rnd = step + if mode == "async" and "training/global_step" in d: + # bare fully_async logs a step's training metrics one parameter version + # later (row step N+1, training/global_step N) and never logs the last + # step's; the rows without it carry the weight sync that ends version N. + rnd = int(d["training/global_step"]) + elif mode == "async" and step == 0: + continue # the weight sync before the first training step + rec = recs.setdefault(rnd, _record(rnd)) + if rnd == step: + _set(p, rec, "t_end_unix", ends.get(step), "console step line (harness time prefix)") gen, key = _first(d, ("timing_s/gen",)) _set(p, rec, "gen_s", gen, key) parts = [k for k in ("timing_s/old_log_prob", "timing_s/update_actor") if d.get(k) is not None] @@ -161,10 +172,13 @@ def parse_verl(metrics_jsonl: str, log_text: str = "", *, samples_per_round: int _set(p, rec, "dropped_samples", dropped, key) generated, key = _first(d, ("fully_async/count/total_generated_samples",)) _set(p, rec, "generated_samples", generated, key) - p.rounds.append(rec) - p.segments_raw.append({k: float(v) for k, v in d.items() if k.startswith("timing_s/") - and isinstance(v, (int, float))}) - p.rounds.sort(key=lambda r: r["round"]) + segs.setdefault(rnd, {}).update({k: float(v) for k, v in d.items() if k.startswith("timing_s/") + and isinstance(v, (int, float))}) + for rec in recs.values(): # a step whose metrics verl never logged still trained one fixed batch + _set(p, rec, "samples", samples_per_round if rec["samples"] is None else rec["samples"], + "config: samples per step (verl trains a fixed batch)") + p.rounds = [recs[r] for r in sorted(recs)] + p.segments_raw = [segs[r] for r in sorted(recs)] _round_s_from_ends(p) na = {"sampler_update_s"} if mode == "sync": diff --git a/tools/eval/efficiency/report.py b/tools/eval/efficiency/report.py index 38c6a9f0..be571fe7 100644 --- a/tools/eval/efficiency/report.py +++ b/tools/eval/efficiency/report.py @@ -202,4 +202,8 @@ def _missing(by_arm: dict[str, list[dict]]) -> list[str]: if a in by_arm: src = by_arm[a][0]["sources"] rows.append(f"- {a}: " + "; ".join(f"{k}: {v}" for k, v in src.items())) + partial = {k: v for s in by_arm[a] for k, v in (s.get("rate_rounds") or {}).items()} + if partial: + rows.append(f" - {a}: rates over the steady rounds that report the value: " + + "; ".join(f"{k} {v}" for k, v in partial.items())) return rows + [""] From c9245e5825193e336aa910e11cc11745f6087aab Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:38:44 +0000 Subject: [PATCH 06/12] tools/eval: bare Miles arms do not shuffle prompts (yeto's Miles argv does not) Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 5 +++++ tools/eval/efficiency/spec.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py index cdd9df04..2cd54e26 100644 --- a/tests/test_tools_eval_efficiency.py +++ b/tests/test_tools_eval_efficiency.py @@ -364,3 +364,8 @@ def test_parse_bare_verl_async_rows_lag_one_version(): assert [r["t_end_unix"] for r in p.rounds] == [101.0, 102.0, 103.0] assert [r["reward_mean"] for r in p.rounds] == [0.5, 0.5, None] assert [r["samples"] for r in p.rounds] == [4, 4, 4] + + +def test_bare_miles_reads_prompts_in_file_order_like_yeto(): + for arm in ("bare-miles-sync", "bare-miles-async"): + assert "--rollout-shuffle" not in spec.bare_script(SPEC, spec.Arm.parse(arm), "r") diff --git a/tools/eval/efficiency/spec.py b/tools/eval/efficiency/spec.py index 5b158ae5..bf40111a 100644 --- a/tools/eval/efficiency/spec.py +++ b/tools/eval/efficiency/spec.py @@ -275,7 +275,9 @@ def miles_bare_argv(spec: EffSpec, mode: str) -> list[str]: "--finetune", "--no-load-optim", "--no-load-rng", "--no-save-optim", "--no-save-rng", # rollout "--prompt-data", f"{DATA_DIR}/train.jsonl", "--input-key", "prompt", "--label-key", "label", - "--apply-chat-template", "--rollout-shuffle", "--rollout-seed", str(spec.seed), + # no --rollout-shuffle (the recipe has it): yeto's Miles argv leaves rollout_shuffle + # False (eff01-yeto-miles-* Miles arg dump), and both arms must read prompts in one order + "--apply-chat-template", "--rollout-seed", str(spec.seed), "--custom-rm-path", "eff_reward.score", "--num-rollout", str(spec.rounds), "--rollout-batch-size", str(spec.groups_per_round), "--n-samples-per-prompt", str(spec.samples_per_group), From 73442ad84a79d71c881bc14358175f3fadf6fad3 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:01:51 +0000 Subject: [PATCH 07/12] verl adapter: fully_async keeps NCCL host buffers on /dev/shm in every Ray worker On Modal one weight-sync process could allocate NCCL host buffers with cuMem while its peer fell back to /dev/shm and then failed to attach (eff01-yeto-verl-async-e; NCCL_DEBUG=INFO in eff01-bare-verl-async-d). NCCL_CUMEM_HOST_ENABLE=0 unless the caller sets it. Co-Authored-By: Claude Opus 5.5 --- tests/test_rl_verl_adapter.py | 24 ++++++++++++++++++++++++ yeto/rl/adapters/verl/verl_main.py | 19 ++++++++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/tests/test_rl_verl_adapter.py b/tests/test_rl_verl_adapter.py index 96c58845..1a417dcd 100644 --- a/tests/test_rl_verl_adapter.py +++ b/tests/test_rl_verl_adapter.py @@ -447,3 +447,27 @@ def test_verl_data_cursor_refuses_what_it_cannot_honour(): cur.seek({"sample_offset": 2}) with pytest.raises(RuntimeError, match="twice"): VerlDataCursor(trainer) + + +def test_fully_async_runtime_env_keeps_nccl_host_buffers_on_shm(monkeypatch): + # S26 E0.1: a cuMem host-buffer sender and a /dev/shm receiver cannot attach on Modal + import sys + import types + + pkg = types.ModuleType("verl.trainer.constants_ppo") + pkg.get_ppo_ray_runtime_env = lambda _x: {"env_vars": {}} + for name in ("verl", "verl.trainer"): + monkeypatch.setitem(sys.modules, name, sys.modules.get(name) or types.ModuleType(name)) + monkeypatch.setitem(sys.modules, "verl.trainer.constants_ppo", pkg) + ray_stub = types.ModuleType("ray") + ray_stub.remote = lambda *a, **k: (lambda cls: cls) + monkeypatch.setitem(sys.modules, "ray", sys.modules.get("ray") or ray_stub) + monkeypatch.setitem(sys.modules, "yeto.rl.adapters.verl.verl_main", None) + monkeypatch.delitem(sys.modules, "yeto.rl.adapters.verl.verl_main") + from yeto.rl.adapters.verl import verl_main + + monkeypatch.delenv(verl_main.CUMEM_HOST_ENV, raising=False) + assert verl_main._runtime_env(fully_async=True)["env_vars"][verl_main.CUMEM_HOST_ENV] == "0" + assert verl_main.CUMEM_HOST_ENV not in verl_main._runtime_env()["env_vars"] + monkeypatch.setenv(verl_main.CUMEM_HOST_ENV, "1") + assert verl_main._runtime_env(fully_async=True)["env_vars"][verl_main.CUMEM_HOST_ENV] == "1" diff --git a/yeto/rl/adapters/verl/verl_main.py b/yeto/rl/adapters/verl/verl_main.py index 106a0093..303ad942 100644 --- a/yeto/rl/adapters/verl/verl_main.py +++ b/yeto/rl/adapters/verl/verl_main.py @@ -57,12 +57,21 @@ def run(self, config, plan: dict): tq.close() -def _runtime_env() -> dict: +# NCCL host buffers on /dev/shm in every process of the disaggregated weight sync. +# On Modal one process could use cuMem host buffers while its peer fell back to +# /dev/shm, and the peer then failed to attach (S26 E0.1 eff01-yeto-verl-async-e, +# eff01-bare-verl-async-d with NCCL_DEBUG=INFO). The caller's own value wins. +CUMEM_HOST_ENV = "NCCL_CUMEM_HOST_ENABLE" + + +def _runtime_env(fully_async: bool = False) -> dict: from verl.trainer.constants_ppo import get_ppo_ray_runtime_env env = get_ppo_ray_runtime_env(None) env.setdefault("env_vars", {}) env["env_vars"]["TRANSFER_QUEUE_ENABLE"] = "1" + if fully_async: + env["env_vars"][CUMEM_HOST_ENV] = os.environ.get(CUMEM_HOST_ENV, "0") for key, value in os.environ.items(): # YETO_ covers YETO_ISLAND_HMAC_KEY (HELLO HMAC in the Ray actor that # opens the syncer client) and YETO_SANDBOX_MODAL_TOKEN_*; no cloud @@ -90,7 +99,7 @@ def _keep_npu_visible_in_cpu_actors() -> None: os.environ.setdefault(NPU_NOSET_ENV, "1") -def _plan_and_ray() -> dict: +def _plan_and_ray(fully_async: bool = False) -> dict: from yeto.island_credential_guard import check_island_credentials check_island_credentials() # secret-handling-hardening D4 (driver, both paths) @@ -98,7 +107,7 @@ def _plan_and_ray() -> dict: if not ray.is_initialized(): _keep_npu_visible_in_cpu_actors() address = os.environ.get("RAY_ADDRESS") or None - ray.init(address=address, runtime_env=_runtime_env(), namespace="yeto-verl") + ray.init(address=address, runtime_env=_runtime_env(fully_async), namespace="yeto-verl") return plan @@ -125,7 +134,7 @@ def main_fully_async(argv: list[str] | None = None) -> None: import verl.trainer.main_ppo as main_ppo from verl.experimental.fully_async_policy import fully_async_main as fa_main - plan = _plan_and_ray() + plan = _plan_and_ray(fully_async=True) def run_ppo(config, task_runner_class=None): from yeto.rl.adapters.verl.fully_async_runner import YetoFullyAsyncTaskRunner @@ -147,7 +156,7 @@ def main_buffer_trainer(argv: list[str] | None = None) -> None: import verl.trainer.main_ppo as main_ppo from verl.experimental.fully_async_policy import fully_async_main as fa_main - plan = _plan_and_ray() + plan = _plan_and_ray(fully_async=True) def run_ppo(config, task_runner_class=None): from yeto.rl.adapters.verl.buffer_trainer_runner import YetoBufferTaskRunner From 0122f4e7aab4b445f9251374a9f039df6e4f1a95 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:23:56 +0000 Subject: [PATCH 08/12] tools/eval: read the yeto+Miles launcher exit code into the parsed run Co-Authored-By: Claude Opus 5.5 --- tools/eval/efficiency/parse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/eval/efficiency/parse.py b/tools/eval/efficiency/parse.py index 066f8d6f..e7cbef96 100644 --- a/tools/eval/efficiency/parse.py +++ b/tools/eval/efficiency/parse.py @@ -417,6 +417,8 @@ def parse_run(run_dir: Path) -> dict[str, Any]: parsed = parse_yeto(_tape_events(raw), mode=mode, framework_parsed=fw, framework_round_offset=1 if plan["framework"] == "verl" else 0) \ if plan["system"] == "yeto" else fw + if meta.get("rc") is None and _read(raw / "rc").strip().lstrip("-").isdigit(): + meta["rc"] = int(_read(raw / "rc")) # miles_local.sh: the launcher's exit code start = meta.get("start_unix") or _utc_file(raw / "start_utc.txt") end = meta.get("end_unix") or _utc_file(raw / "end_utc.txt") return {"arm": plan["arm"], "run_id": plan["run_id"], "fingerprint": plan["fingerprint"], "mode": mode, From e0f833dccd1bfed3524756fcf2c71183f6443e19 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:34:51 +0000 Subject: [PATCH 09/12] miles adapter: opt-in gradient-accumulation fusion; generic recipe trains without dropout --rl-gradient-accumulation-fusion lets a run keep Megatron's fused wgrad (the Miles recipes' default); off keeps the historical argv. The generic recipe passed no dropout, so Megatron's 0.1 applied during RL training while the Miles recipes pass 0.0; it now passes 0.0 as the Qwen3.5 recipe does. Co-Authored-By: Claude Opus 5.5 --- tests/test_rl_miles_adapter_config.py | 18 ++++++++++++++++++ yeto/launch/rl_island_task.py | 2 ++ yeto/rl/adapters/miles/config.py | 8 +++++++- yeto/rl/adapters/miles/run_flags.py | 6 ++++++ yeto/rl/engine/run_config.py | 4 ++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/test_rl_miles_adapter_config.py b/tests/test_rl_miles_adapter_config.py index 503fe2e2..7cbe30d9 100644 --- a/tests/test_rl_miles_adapter_config.py +++ b/tests/test_rl_miles_adapter_config.py @@ -795,3 +795,21 @@ def test_dynamic_batching_refused_for_bshd_gated_delta_net_models(): recompute_only = dataclasses.replace( cfg, model_recipe=recipe, train_memory=rc.TrainMemoryConfig(activation_recompute=True)) assert "--recompute-granularity" in mc.translate_run_config(recompute_only, AlgorithmSpec()).argv + + +def test_gradient_accumulation_fusion_is_opt_in(): + # S26 E0.1: the Miles recipes train with the fusion on; yeto keeps it off by default + off = " ".join(mc.translate_run_config(make_config(), AlgorithmSpec()).argv) + assert "--no-gradient-accumulation-fusion" in off + cfg = dataclasses.replace(make_config(), train_memory=rc.TrainMemoryConfig(gradient_accumulation_fusion=True)) + assert "--no-gradient-accumulation-fusion" not in mc.translate_run_config(cfg, AlgorithmSpec()).argv + assert rc._train_memory(SimpleNamespace(seq_len=8, rl_gradient_accumulation_fusion=True)) \ + .gradient_accumulation_fusion + + +def test_generic_recipe_trains_without_dropout(): + cfg = make_config() + if cfg.model_recipe.name != rc.RECIPE_GENERIC: + cfg = sub(cfg, "model_recipe", name=rc.RECIPE_GENERIC) + joined = " ".join(mc.translate_run_config(cfg, AlgorithmSpec()).argv) + assert "--attention-dropout 0.0 --hidden-dropout 0.0" in joined diff --git a/yeto/launch/rl_island_task.py b/yeto/launch/rl_island_task.py index 1185f4c9..ce1ceb50 100644 --- a/yeto/launch/rl_island_task.py +++ b/yeto/launch/rl_island_task.py @@ -153,6 +153,8 @@ def make_miles_island_task( + f" --inner-lr {args.inner_lr}" f" --seq-len {args.seq_len}" + (" --rl-activation-recompute" if getattr(args, "rl_activation_recompute", False) else "") + + (" --rl-gradient-accumulation-fusion" + if getattr(args, "rl_gradient_accumulation_fusion", False) else "") + (f" --rl-max-tokens-per-gpu {int(args.rl_max_tokens_per_gpu)}" if getattr(args, "rl_max_tokens_per_gpu", None) is not None else "") + (f" --rl-optimizer-offload-fraction {float(args.rl_optimizer_offload_fraction)!r}" diff --git a/yeto/rl/adapters/miles/config.py b/yeto/rl/adapters/miles/config.py index dadbb341..1af82e71 100644 --- a/yeto/rl/adapters/miles/config.py +++ b/yeto/rl/adapters/miles/config.py @@ -472,6 +472,7 @@ def _check_sglang_tp(value, config): # translated below (Megatron full recompute / Miles dynamic batching) "train_memory.activation_recompute": _ok, "train_memory.max_tokens_per_gpu": _ok, + "train_memory.gradient_accumulation_fusion": _ok, "train_memory.optimizer_offload_fraction": _ok, **{f"sample_buffer.{name}": _ok for name in ("role", "store", "max_backlog_groups", "wait_s", "min_round_s")}, @@ -781,6 +782,7 @@ def translate_run_config( from yeto.rl.engine.run_config import ( RECIPE_DEEPSEEK_V4_FLASH, + RECIPE_GENERIC, RECIPE_GLM5_NEXT, RECIPE_QWEN3_5, RECIPE_QWEN3_8_NEXT, @@ -864,6 +866,10 @@ def translate_run_config( model_recipe_values = deepseek_v4.recipe_argv(trainable) else: model_name = recipe.provider_class + if recipe.name == RECIPE_GENERIC: + # Megatron's default dropout is 0.1; the Miles recipes train RL with + # none (the policy's train-time log-probs must match the rollout's) + model_recipe_values = ["--attention-dropout", "0.0", "--hidden-dropout", "0.0"] values: list[str] = [ "train.py", @@ -938,7 +944,7 @@ def translate_run_config( "--accumulate-allreduce-grads-in-fp32", "--attention-softmax-in-fp32", "--attention-backend", recipe.training_attention_backend, - "--no-gradient-accumulation-fusion", + *(() if config.train_memory.gradient_accumulation_fusion else ("--no-gradient-accumulation-fusion",)), "--bf16", "--no-load-optim", "--no-load-rng", diff --git a/yeto/rl/adapters/miles/run_flags.py b/yeto/rl/adapters/miles/run_flags.py index 8a9f5202..6e4a5af0 100644 --- a/yeto/rl/adapters/miles/run_flags.py +++ b/yeto/rl/adapters/miles/run_flags.py @@ -175,6 +175,12 @@ "at most this many tokens per GPU (must be >= --seq-len). Default: fixed " "micro batches", ), + shared_flag( + "--rl-gradient-accumulation-fusion", + action="store_true", + help="ports: Megatron gradient-accumulation fusion (the Miles recipes' " + "default). Off by default (--no-gradient-accumulation-fusion)", + ), shared_flag( "--rl-optimizer-offload-fraction", type=float, diff --git a/yeto/rl/engine/run_config.py b/yeto/rl/engine/run_config.py index 616e588a..90abd7dc 100644 --- a/yeto/rl/engine/run_config.py +++ b/yeto/rl/engine/run_config.py @@ -255,6 +255,9 @@ class TrainMemoryConfig: max_tokens_per_gpu: int | None = None # Share of the optimizer state kept and updated on the host (0 = none). optimizer_offload_fraction: float = 0.0 + # Megatron gradient-accumulation fusion (fused wgrad into main_grad; needs + # APEX/TE fused kernels). Off keeps the historical argv. + gradient_accumulation_fusion: bool = False @dataclass(frozen=True) @@ -1094,6 +1097,7 @@ def _train_memory(args) -> TrainMemoryConfig: activation_recompute=bool(getattr(args, "rl_activation_recompute", False)), max_tokens_per_gpu=int(max_tokens) if max_tokens is not None else None, optimizer_offload_fraction=offload, + gradient_accumulation_fusion=bool(getattr(args, "rl_gradient_accumulation_fusion", False)), ) From 6a4263e24d62eabbe7cc991a5422375e340ac246 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:35:46 +0000 Subject: [PATCH 10/12] tools/eval: yeto+Miles arm takes the bare Miles recipe settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No deterministic inference, constant LR, full recompute, dynamic batching, gradient-accumulation fusion, and an algorithm spec with the recipe's clip range (0.2 / 0.28); S26 GOAL §6 makes the bare framework's settings the reference. Co-Authored-By: Claude Opus 5.5 --- tools/eval/efficiency/algo-eff-miles.json | 53 +++++++++++++++++++++++ tools/eval/efficiency/spec.py | 20 +++++++-- 2 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 tools/eval/efficiency/algo-eff-miles.json diff --git a/tools/eval/efficiency/algo-eff-miles.json b/tools/eval/efficiency/algo-eff-miles.json new file mode 100644 index 00000000..f45d2e97 --- /dev/null +++ b/tools/eval/efficiency/algo-eff-miles.json @@ -0,0 +1,53 @@ +{ + "advantage": { + "estimator": "grpo", + "reward_binary": false, + "reward_postprocess": null, + "rewards_normalization": true, + "std_normalization": true, + "whiten": false + }, + "correction": { + "batch_normalize": false, + "function": null, + "level": "token", + "lower": 0.0, + "method": "tis", + "mismatch_metrics": false, + "opsm_delta": null, + "ratio_mask": "none", + "upper": 2.0, + "use_rollout_logprobs": false, + "veto": null + }, + "entropy_coef": 0.0, + "execution": { + "group_required": true, + "max_policy_staleness": 2, + "needs_critic": false, + "needs_rollout_logprobs": false + }, + "kl": { + "coef": null, + "estimator": null, + "placement": "none", + "unbiased": false + }, + "loss": { + "aggregation": "default", + "custom_loss": null, + "eps_clip": 0.2, + "eps_clip_c": null, + "eps_clip_high": 0.28, + "reducer": null, + "variant": "policy_loss" + }, + "plugins": [], + "sampling": { + "filter": null, + "max_replacements": null, + "over_sampling_batch_size": null, + "overlong_filter": false + }, + "schema": "yeto-rl-algorithm-spec-v2" +} \ No newline at end of file diff --git a/tools/eval/efficiency/spec.py b/tools/eval/efficiency/spec.py index bf40111a..1d1de74b 100644 --- a/tools/eval/efficiency/spec.py +++ b/tools/eval/efficiency/spec.py @@ -300,7 +300,7 @@ def miles_bare_argv(spec: EffSpec, mode: str) -> list[str]: "--tensor-model-parallel-size", "1", "--pipeline-model-parallel-size", "1", "--context-parallel-size", "1", "--expert-model-parallel-size", "1", "--expert-tensor-parallel-size", "1", "--recompute-granularity", "full", "--recompute-method", "uniform", "--recompute-num-layers", "1", - "--use-dynamic-batch-size", "--max-tokens-per-gpu", str(max(9216, spec.seq_len)), + "--use-dynamic-batch-size", "--max-tokens-per-gpu", str(max_tokens_per_gpu(spec)), "--attention-dropout", "0.0", "--hidden-dropout", "0.0", "--accumulate-allreduce-grads-in-fp32", "--attention-softmax-in-fp32", # yeto's generic recipe (Qwen3) trains with "unfused": TE's flash path picks the @@ -327,9 +327,16 @@ def miles_bare_script(mode: str) -> str: return "train.py" if mode == "sync" else "train_async.py" +def max_tokens_per_gpu(spec: EffSpec) -> int: + """Miles dynamic-batching budget (run_qwen3_4b.py: 9216), never below one sample.""" + return max(9216, spec.seq_len) + + # ====================================================================== yeto arms YETO_WORKDIR = "/root/sky_workdir" # modal_verl.WORKDIR: the repo inside the yeto+verl container ALGORITHM_SPEC = "examples/rl_algorithms/decoupled-ppo-c2.json" # token TIS 2.0, max_policy_staleness 2 +# the same with the bare Miles recipe's clip range (--eps-clip 0.2 --eps-clip-high 0.28) +MILES_ALGORITHM_SPEC = "tools/eval/efficiency/algo-eff-miles.json" def yeto_verl_argv(spec: EffSpec, mode: str, tape: str) -> list[str]: """``island_entry`` arguments of the yeto+verl arm (the T-EVAL E1 shape).""" argv = ["-m", "yeto.rl.adapters.verl.island_entry", "--model", spec.model, "--model-revision", @@ -351,7 +358,7 @@ def yeto_verl_argv(spec: EffSpec, mode: str, tape: str) -> list[str]: def yeto_miles_launcher_args(spec: EffSpec, mode: str, run_id: str) -> str: """The yeto launcher arguments of the yeto+Miles arm (``tools/eval/miles_local.sh`` ARGS).""" - from ..miles import DATA, SPEC, STORE_VOLUME, TAPE_VOLUME + from ..miles import DATA, STORE_VOLUME, TAPE_VOLUME card = {"A10G": "a10g", "H100!": "h100", "L4": "l4"}[spec.gpu] c = cards(mode) @@ -366,8 +373,13 @@ def yeto_miles_launcher_args(spec: EffSpec, mode: str, run_id: str) -> str: f"--rollout-max-response-len {spec.max_response_len} --seq-len {spec.seq_len} " f"--inner-lr {spec.lr} --seed {spec.seed} --sglang-mem-fraction-static {spec.mem_fraction} " f"--gpu modal:{c['total']}x{card} --rl-single-island-no-sync --total-steps {spec.rounds} " - # TIS (upper 2.0) in both modes, as in the bare arms; tolerates staleness 2 >= max_policy_age - f"--trust-remote-code {SPEC}") + # Miles recipe settings of the bare arm (S26 GOAL §6: the bare framework's official + # settings win): no deterministic inference, constant LR, full recompute, dynamic + # batching, gradient-accumulation fusion; dropout 0 is the generic recipe's own + "--no-sglang-deterministic-inference --rl-lr-schedule constant --rl-activation-recompute " + f"--rl-max-tokens-per-gpu {max_tokens_per_gpu(spec)} --rl-gradient-accumulation-fusion " + # TIS (upper 2.0) and eps_clip_high 0.28 as in the bare arms; staleness 2 >= max_policy_age + f"--trust-remote-code --rl-algorithm-spec {MILES_ALGORITHM_SPEC}") if kwargs: args += f" --apply-chat-template-kwargs {shlex.quote(kwargs)}" if mode == "async": From ff893bcd146969ed20458cbeeb72b199c93e55c9 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:38:14 +0000 Subject: [PATCH 11/12] tools/eval: read only the run's own tapes (skip the repo copy and the store) miles_local.sh pulls a copy of the repo whose test fixtures are tapes of other runs; their rl_local_round rows replaced the run's rewards (all four yeto+Miles runs showed the same fixture curve with reward 0 in round 0). Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 8 ++++++++ tools/eval/efficiency/parse.py | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py index 2cd54e26..c5f19024 100644 --- a/tests/test_tools_eval_efficiency.py +++ b/tests/test_tools_eval_efficiency.py @@ -369,3 +369,11 @@ def test_parse_bare_verl_async_rows_lag_one_version(): def test_bare_miles_reads_prompts_in_file_order_like_yeto(): for arm in ("bare-miles-sync", "bare-miles-async"): assert "--rollout-shuffle" not in spec.bare_script(SPEC, spec.Arm.parse(arm), "r") + + +def test_tape_events_skip_the_repo_copy_and_the_store(tmp_path): + row = {"event": "rl_local_round", "local_round_id": 1, "time_unix": 1.0} + for sub in ("tape-direct/l0", "yeto/tests/fixtures", "store/x"): + (tmp_path / sub).mkdir(parents=True) + (tmp_path / sub / "rl-island-0.jsonl").write_text(json.dumps({**row, "src": sub}) + "\n") + assert [r.get("src") for r in parse._tape_events(tmp_path)] == ["tape-direct/l0"] diff --git a/tools/eval/efficiency/parse.py b/tools/eval/efficiency/parse.py index e7cbef96..59f9ee15 100644 --- a/tools/eval/efficiency/parse.py +++ b/tools/eval/efficiency/parse.py @@ -383,10 +383,15 @@ def _read(path: Path) -> str: return path.read_text(errors="replace") if path.is_file() else "" +NOT_RUN_TAPES = ("yeto", "store") + + def _tape_events(raw: Path) -> list[dict[str, Any]]: seen, rows = set(), [] for path in sorted(raw.rglob("*.jsonl")): - if "verl-" in path.name: + # miles_local.sh pulls a copy of the repo (raw/yeto, whose test fixtures are + # tapes of other runs) and the checkpoint store next to the run's own tapes + if "verl-" in path.name or path.relative_to(raw).parts[0] in NOT_RUN_TAPES: continue for row in tape_rows(_read(path)): key = json.dumps(row, sort_keys=True, default=str) # the same event pulled from two sources From ba179531cb1012c47464825ff39944f28371aafa Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:39:05 +0000 Subject: [PATCH 12/12] tools/eval: verl fully_async generation time from the rollouter, else missing The trainer's timing_s/gen and yeto's driver generate phase only pull finished samples from the rollouter queue. Bare verl logs the rollouter's active time per parameter version; yeto logs none per round, so its value stays missing. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools_eval_efficiency.py | 1 + tools/eval/efficiency/parse.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_tools_eval_efficiency.py b/tests/test_tools_eval_efficiency.py index c5f19024..2cd3821c 100644 --- a/tests/test_tools_eval_efficiency.py +++ b/tests/test_tools_eval_efficiency.py @@ -364,6 +364,7 @@ def test_parse_bare_verl_async_rows_lag_one_version(): assert [r["t_end_unix"] for r in p.rounds] == [101.0, 102.0, 103.0] assert [r["reward_mean"] for r in p.rounds] == [0.5, 0.5, None] assert [r["samples"] for r in p.rounds] == [4, 4, 4] + assert [r["gen_s"] for r in p.rounds] == [1.0, 1.0, 1.0] # rollouter active time, not the queue pull def test_bare_miles_reads_prompts_in_file_order_like_yeto(): diff --git a/tools/eval/efficiency/parse.py b/tools/eval/efficiency/parse.py index 59f9ee15..f3b2f8ee 100644 --- a/tools/eval/efficiency/parse.py +++ b/tools/eval/efficiency/parse.py @@ -146,7 +146,9 @@ def parse_verl(metrics_jsonl: str, log_text: str = "", *, samples_per_round: int rec = recs.setdefault(rnd, _record(rnd)) if rnd == step: _set(p, rec, "t_end_unix", ends.get(step), "console step line (harness time prefix)") - gen, key = _first(d, ("timing_s/gen",)) + # fully_async: the trainer's timing_s/gen only pulls finished samples from the + # queue; the rollouter's active time in the version window is the generation + gen, key = _first(d, ("fully_async/rollouter/active_time",) if mode == "async" else ("timing_s/gen",)) _set(p, rec, "gen_s", gen, key) parts = [k for k in ("timing_s/old_log_prob", "timing_s/update_actor") if d.get(k) is not None] if parts: @@ -283,7 +285,7 @@ def tape_rows(text: str) -> list[dict[str, Any]]: def parse_yeto(events: list[dict[str, Any]], *, mode: str, framework_parsed: Parsed | None = None, - framework_round_offset: int = 0) -> Parsed: + framework_round_offset: int = 0, skip: frozenset[str] = frozenset()) -> Parsed: """yeto tape -> rounds; ``framework_parsed`` (the framework's own log in the same run, e.g. Miles perf lines) fills fields the tape does not carry. Its round ``rollout_id + framework_round_offset`` is the tape's ``rollout_id``: @@ -317,7 +319,8 @@ def parse_yeto(events: list[dict[str, Any]], *, mode: str, framework_parsed: Par seg = dict(spans.get(rid) or phases.get(rid) or {}) src = "rl_timeline_span" if rid in spans else "rl_driver_phase (marker to next marker)" for task, name in YETO_SEGMENT.items(): - _set(p, rec, name, seg.get(task), f"{src} {task}") + if name not in skip: + _set(p, rec, name, seg.get(task), f"{src} {task}") if mode == "async": _set(p, rec, "wait_s", seg.get("wait"), f"{src} kind=wait") _set(p, rec, "gen_tokens", lr.get("action_tokens"), "rl_local_round.action_tokens") @@ -328,7 +331,8 @@ def parse_yeto(events: list[dict[str, Any]], *, mode: str, framework_parsed: Par _set(p, rec, "dropped_samples", drops.get(rid), "rl_rollout_carry_over.over_age_discarded_groups (groups)") f = fw.get(rid + framework_round_offset, {}) for name in ROUND_FIELDS: # what the tape lacks, from the framework's own log of the same run - if rec[name] is None and f.get(name) is not None and name not in ("round", "t_end_unix", "round_s"): + if rec[name] is None and f.get(name) is not None and name not in ("round", "t_end_unix", "round_s") \ + and name not in skip: _set(p, rec, name, f[name], "framework log: " + framework_parsed.sources[name]) p.rounds.append(rec) p.segments_raw.append({k: round(v, 3) for k, v in seg.items()}) @@ -419,7 +423,11 @@ def parse_run(run_dir: Path) -> dict[str, Any]: else: log = _read(arm_dir / "main.log") or _read(raw / "launch.ts.log") fw = parse_miles(log, rollout_cards=c["rollout"], mode=mode) - parsed = parse_yeto(_tape_events(raw), mode=mode, framework_parsed=fw, + # yeto+verl fully_async: the driver's generate phase (and verl's timing_s/gen) only + # pull finished samples from the rollouter queue, and yeto logs no per-round + # rollouter time, so generation time stays missing + skip = frozenset({"gen_s"}) if plan["framework"] == "verl" and mode == "async" else frozenset() + parsed = parse_yeto(_tape_events(raw), mode=mode, framework_parsed=fw, skip=skip, framework_round_offset=1 if plan["framework"] == "verl" else 0) \ if plan["system"] == "yeto" else fw if meta.get("rc") is None and _read(raw / "rc").strip().lstrip("-").isdigit():