From 67663c1e9fa5bc2e88d83e105e68f3441ba7c44d Mon Sep 17 00:00:00 2001 From: Kinflou Date: Sun, 8 Nov 2026 13:10:07 +0800 Subject: [PATCH 1/2] reference/languages/typescript: bench and property reports from the reports branch --- docs/docs/reference/languages/typescript.md | 20 ++++++++++++++++++-- scripts/sync-language-support.py | 16 ++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/docs/reference/languages/typescript.md b/docs/docs/reference/languages/typescript.md index f5f3db6..3d9f7be 100644 --- a/docs/docs/reference/languages/typescript.md +++ b/docs/docs/reference/languages/typescript.md @@ -92,8 +92,14 @@ tracked, each with the issue that will fix it. - Pull requests run a short suite; a nightly run takes 20,000 cases per property and opens an issue on a divergence outside the known classes. -The property suite doesn't write a results file, so there are no run counts to -show here. Coverage-guided fuzzing (Jazzer.js) isn't set up yet. +The nightly run publishes its verdicts as `typescript.fuzz.json` on the +repository's [`reports`](https://github.com/ComlineProject/comline-typescript/tree/reports) +branch. The latest, fetched when this site was built: + +--8<-- "_generated/reports/typescript.fuzz.md" + +Every nightly run, charted: [Observatory → Fuzzing](https://comlineproject.github.io/observatory/fuzz/typescript-fuzz/). +Coverage-guided fuzzing (Jazzer.js) isn't set up yet. ### Divergences @@ -114,4 +120,14 @@ Where `typescript` and the reference are known to answer differently, from [`kno ### Benchmarks +Instruction counts for the shared workloads of the runtime's conformance +corpus, under the same ids as Rust's, measured after warm-up (the median of +five runs). Every push to `main` publishes `typescript.bench.json` on the +[`reports`](https://github.com/ComlineProject/comline-typescript/tree/reports) +branch. The latest, fetched when this site was built: + --8<-- "_generated/reports/typescript.bench.md" + +Every published run, and the ratio to Rust per workload: +[Observatory → Benchmarks](https://comlineproject.github.io/observatory/bench/typescript-bench/) +and [ratio to Rust](https://comlineproject.github.io/observatory/ratio/). diff --git a/scripts/sync-language-support.py b/scripts/sync-language-support.py index be56f41..75d0a04 100644 --- a/scripts/sync-language-support.py +++ b/scripts/sync-language-support.py @@ -87,8 +87,8 @@ # None is `--runtime-rev`. A runtime with no entry has none to show (Rust is # the reference the others are compared against, so it has no divergences # file). A report that isn't there yet (HTTP 404) is shown as "not published -# yet". Rust's are on the runtime repository's `reports` branch: its CI -# commits them there from `main` (bench, every push) and the nightly fuzz run, +# yet". Each runtime's are on its repository's `reports` branch: its CI +# commits them there from `main` (bench, every push) and its nightly run, # since `main` itself only takes pull requests. REPORTS_BRANCH = "reports" DIVERGENCES = { @@ -96,10 +96,11 @@ } BENCHES = { "rust": ("runtime", REPORTS_BRANCH, "rust.bench.json"), - "typescript": ("comline-typescript", None, "runtime/conformance-bench.json"), + "typescript": ("comline-typescript", REPORTS_BRANCH, "typescript.bench.json"), } FUZZ = { "rust": ("runtime", REPORTS_BRANCH, "rust.fuzz.json"), + "typescript": ("comline-typescript", REPORTS_BRANCH, "typescript.fuzz.json"), } REPORT_URL = "https://github.com/ComlineProject/{repo}/blob/{rev}/{path}" VERIFY_NOTICE = ( @@ -320,10 +321,13 @@ def number(n: int | None) -> str: f"From the run of {report['date'][:10]} at runtime commit `{report['commit'][:7]}`, " f"with `{report['toolchain']}` ([run]({report['run']}), " f"[`{path.rsplit('/', 1)[-1]}`]({source})).", - "", - "| Target | Seconds | Executions | Corpus | Findings |", - "|---|---:|---:|---:|---:|", ] + if report["targets"]: + lines += [ + "", + "| Target | Seconds | Executions | Corpus | Findings |", + "|---|---:|---:|---:|---:|", + ] for target, t in sorted(report["targets"].items()): findings = "didn't run" if t.get("error") else number(t["findings"]) lines.append( From 56154023007662ac18573041cfc0dc66dab9f94f Mon Sep 17 00:00:00 2001 From: Kinflou Date: Sun, 8 Nov 2026 13:40:07 +0800 Subject: [PATCH 2/2] TypeScript bench reports: two metrics per benchmark (format 2) --- docs/docs/reference/languages/typescript.md | 6 ++-- scripts/sync-language-support.py | 38 +++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/docs/reference/languages/typescript.md b/docs/docs/reference/languages/typescript.md index 3d9f7be..49657b3 100644 --- a/docs/docs/reference/languages/typescript.md +++ b/docs/docs/reference/languages/typescript.md @@ -121,8 +121,10 @@ Where `typescript` and the reference are known to answer differently, from [`kno ### Benchmarks Instruction counts for the shared workloads of the runtime's conformance -corpus, under the same ids as Rust's, measured after warm-up (the median of -five runs). Every push to `main` publishes `typescript.bench.json` on the +corpus, under the same ids as Rust's, each the median of five runs, in two +tiers: with the JIT's optimizing tier off (exact, so it tracks this runtime's +own changes) and on, after warm-up (closest to real use, and what the ratio to +Rust compares). Every push to `main` publishes `typescript.bench.json` on the [`reports`](https://github.com/ComlineProject/comline-typescript/tree/reports) branch. The latest, fetched when this site was built: diff --git a/scripts/sync-language-support.py b/scripts/sync-language-support.py index 75d0a04..cde8f61 100644 --- a/scripts/sync-language-support.py +++ b/scripts/sync-language-support.py @@ -103,6 +103,15 @@ "typescript": ("comline-typescript", REPORTS_BRANCH, "typescript.fuzz.json"), } REPORT_URL = "https://github.com/ComlineProject/{repo}/blob/{rev}/{path}" +# Column titles and notes for a bench report's metrics (format 2). +BENCH_METRICS = { + "instructions": "Instructions", + "instructions-opt": "Instructions, optimized", +} +BENCH_METRIC_NOTES = { + "instructions": "instructions executed with the JIT's optimizing tier off: exact run to run, so they track the runtime's own changes.", + "instructions-opt": "the same after warm-up with the optimizing tier on: closest to real use, and what the ratio to Rust compares.", +} VERIFY_NOTICE = ( "" @@ -293,18 +302,35 @@ def bench_table(name: str, report: dict | None) -> str: source = REPORT_URL.format(repo=repo, rev=rev or "main", path=path) if report is None: return not_published("benchmark results", repo, rev, path) - metric = {"instructions": "instructions executed"}.get(report.get("metric"), report.get("metric", "value")) + file = f"[`{path.rsplit('/', 1)[-1]}`]({source})" + head = f"Measured at commit `{report['commit'][:7]}` with `{report['toolchain']}`" + if report.get("format", 1) == 1: + # One metric per bench: `{"ir": N}`. + metric = {"instructions": "instructions executed"}.get(report.get("metric"), report.get("metric", "value")) + columns = [("ir", "Instructions")] + intro = f"{head}, in {metric} (deterministic, so they compare exactly), from {file}." + else: + # Several metrics per bench: `{"": {"value", "spread"?}}`. + names = sorted({m for b in report["benches"].values() for m in b}) + columns = [(m, BENCH_METRICS.get(m, f"`{m}`")) for m in names] + intro = f"{head}, from {file}. " + " ".join( + f"**{label}:** {BENCH_METRIC_NOTES[m]}" for m, label in columns if m in BENCH_METRIC_NOTES + ) lines = [ - f"Measured at runtime commit `{report['commit'][:7]}` with `{report['toolchain']}`, " - f"in {metric} (deterministic, so they compare exactly), from [`{path.rsplit('/', 1)[-1]}`]({source}).", + intro, "", f'??? note "All {len(report["benches"])} benchmarks"', "", - " | Benchmark | Instructions |", - " |---|---:|", + " | Benchmark | " + " | ".join(label for _, label in columns) + " |", + " |---|" + "---:|" * len(columns), ] for bench, result in sorted(report["benches"].items()): - lines.append(f" | `{bench}` | {result['ir']:,} |") + cells = [] + for m, _ in columns: + v = result.get(m) + v = v.get("value") if isinstance(v, dict) else v + cells.append("—" if v is None else f"{v:,}") + lines.append(f" | `{bench}` | " + " | ".join(cells) + " |") return "\n".join(lines)