Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions docs/docs/reference/languages/typescript.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,14 @@ tracked, each with the issue that will fix it.
- Pull requests run a short suite; a nightly run takes 20,000 cases per
property and opens an issue on a divergence outside the known classes.

The property suite doesn't write a results file, so there are no run counts to
show here. Coverage-guided fuzzing (Jazzer.js) isn't set up yet.
The nightly run publishes its verdicts as `typescript.fuzz.json` on the
repository's [`reports`](https://github.com/ComlineProject/comline-typescript/tree/reports)
branch. The latest, fetched when this site was built:

--8<-- "_generated/reports/typescript.fuzz.md"

Every nightly run, charted: [Observatory → Fuzzing](https://comlineproject.github.io/observatory/fuzz/typescript-fuzz/).
Coverage-guided fuzzing (Jazzer.js) isn't set up yet.

### Divergences

Expand All @@ -114,4 +120,16 @@ Where `typescript` and the reference are known to answer differently, from [`kno

### Benchmarks

Instruction counts for the shared workloads of the runtime's conformance
corpus, under the same ids as Rust's, each the median of five runs, in two
tiers: with the JIT's optimizing tier off (exact, so it tracks this runtime's
own changes) and on, after warm-up (closest to real use, and what the ratio to
Rust compares). Every push to `main` publishes `typescript.bench.json` on the
[`reports`](https://github.com/ComlineProject/comline-typescript/tree/reports)
branch. The latest, fetched when this site was built:

--8<-- "_generated/reports/typescript.bench.md"

Every published run, and the ratio to Rust per workload:
[Observatory → Benchmarks](https://comlineproject.github.io/observatory/bench/typescript-bench/)
and [ratio to Rust](https://comlineproject.github.io/observatory/ratio/).
54 changes: 42 additions & 12 deletions scripts/sync-language-support.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,21 +87,31 @@
# None is `--runtime-rev`. A runtime with no entry has none to show (Rust is
# the reference the others are compared against, so it has no divergences
# file). A report that isn't there yet (HTTP 404) is shown as "not published
# yet". Rust's are on the runtime repository's `reports` branch: its CI
# commits them there from `main` (bench, every push) and the nightly fuzz run,
# yet". Each runtime's are on its repository's `reports` branch: its CI
# commits them there from `main` (bench, every push) and its nightly run,
# since `main` itself only takes pull requests.
REPORTS_BRANCH = "reports"
DIVERGENCES = {
"typescript": ("comline-typescript", None, "runtime/known-divergences.json"),
}
BENCHES = {
"rust": ("runtime", REPORTS_BRANCH, "rust.bench.json"),
"typescript": ("comline-typescript", None, "runtime/conformance-bench.json"),
"typescript": ("comline-typescript", REPORTS_BRANCH, "typescript.bench.json"),
}
FUZZ = {
"rust": ("runtime", REPORTS_BRANCH, "rust.fuzz.json"),
"typescript": ("comline-typescript", REPORTS_BRANCH, "typescript.fuzz.json"),
}
REPORT_URL = "https://github.com/ComlineProject/{repo}/blob/{rev}/{path}"
# Column titles and notes for a bench report's metrics (format 2).
BENCH_METRICS = {
"instructions": "Instructions",
"instructions-opt": "Instructions, optimized",
}
BENCH_METRIC_NOTES = {
"instructions": "instructions executed with the JIT's optimizing tier off: exact run to run, so they track the runtime's own changes.",
"instructions-opt": "the same after warm-up with the optimizing tier on: closest to real use, and what the ratio to Rust compares.",
}
VERIFY_NOTICE = (
"<!-- Rendered from each runtime's verification reports by "
"scripts/sync-language-support.py - don't edit by hand. -->"
Expand Down Expand Up @@ -292,18 +302,35 @@ def bench_table(name: str, report: dict | None) -> str:
source = REPORT_URL.format(repo=repo, rev=rev or "main", path=path)
if report is None:
return not_published("benchmark results", repo, rev, path)
metric = {"instructions": "instructions executed"}.get(report.get("metric"), report.get("metric", "value"))
file = f"[`{path.rsplit('/', 1)[-1]}`]({source})"
head = f"Measured at commit `{report['commit'][:7]}` with `{report['toolchain']}`"
if report.get("format", 1) == 1:
# One metric per bench: `{"ir": N}`.
metric = {"instructions": "instructions executed"}.get(report.get("metric"), report.get("metric", "value"))
columns = [("ir", "Instructions")]
intro = f"{head}, in {metric} (deterministic, so they compare exactly), from {file}."
else:
# Several metrics per bench: `{"<metric>": {"value", "spread"?}}`.
names = sorted({m for b in report["benches"].values() for m in b})
columns = [(m, BENCH_METRICS.get(m, f"`{m}`")) for m in names]
intro = f"{head}, from {file}. " + " ".join(
f"**{label}:** {BENCH_METRIC_NOTES[m]}" for m, label in columns if m in BENCH_METRIC_NOTES
)
lines = [
f"Measured at runtime commit `{report['commit'][:7]}` with `{report['toolchain']}`, "
f"in {metric} (deterministic, so they compare exactly), from [`{path.rsplit('/', 1)[-1]}`]({source}).",
intro,
"",
f'??? note "All {len(report["benches"])} benchmarks"',
"",
" | Benchmark | Instructions |",
" |---|---:|",
" | Benchmark | " + " | ".join(label for _, label in columns) + " |",
" |---|" + "---:|" * len(columns),
]
for bench, result in sorted(report["benches"].items()):
lines.append(f" | `{bench}` | {result['ir']:,} |")
cells = []
for m, _ in columns:
v = result.get(m)
v = v.get("value") if isinstance(v, dict) else v
cells.append("—" if v is None else f"{v:,}")
lines.append(f" | `{bench}` | " + " | ".join(cells) + " |")
return "\n".join(lines)


Expand All @@ -320,10 +347,13 @@ def number(n: int | None) -> str:
f"From the run of {report['date'][:10]} at runtime commit `{report['commit'][:7]}`, "
f"with `{report['toolchain']}` ([run]({report['run']}), "
f"[`{path.rsplit('/', 1)[-1]}`]({source})).",
"",
"| Target | Seconds | Executions | Corpus | Findings |",
"|---|---:|---:|---:|---:|",
]
if report["targets"]:
lines += [
"",
"| Target | Seconds | Executions | Corpus | Findings |",
"|---|---:|---:|---:|---:|",
]
for target, t in sorted(report["targets"].items()):
findings = "didn't run" if t.get("error") else number(t["findings"])
lines.append(
Expand Down
Loading