From 3e7277f9ee728aba2fe5e471276993c1051a40fd Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Mon, 2 Nov 2026 17:04:50 +0800 Subject: [PATCH 1/3] Add strict H3 Ref2VA media integrity validation and lossless plan Agent-assisted-by: Codex Signed-off-by: amy-why-3459 --- evidence/h3-ref2va-lossless-293/README.md | 73 ++++++ .../h3-ref2va-lossless-293/patch/README.md | 11 + .../plan/integrity.json | 11 + .../plan/screening.json | 53 ++++ .../results/validator-regression.json | 51 ++++ .../validator/compare.py | 230 ++++++++++++++++++ .../validator/gates.json | 14 ++ .../validator/integrity.py | 100 ++++++++ .../h3-ref2va-lossless-293/validator/smoke.py | 90 +++++++ .../validator/sources.json | 17 ++ .../validator/upstream_metrics.py | 100 ++++++++ .../validator/weights.json | 4 + tests/test_evidence_inventory.py | 4 +- tests/test_h3_lossless_validator.py | 61 +++++ tools/evidence_inventory.json | 5 + 15 files changed, 822 insertions(+), 2 deletions(-) create mode 100644 evidence/h3-ref2va-lossless-293/README.md create mode 100644 evidence/h3-ref2va-lossless-293/patch/README.md create mode 100644 evidence/h3-ref2va-lossless-293/plan/integrity.json create mode 100644 evidence/h3-ref2va-lossless-293/plan/screening.json create mode 100644 evidence/h3-ref2va-lossless-293/results/validator-regression.json create mode 100644 evidence/h3-ref2va-lossless-293/validator/compare.py create mode 100644 evidence/h3-ref2va-lossless-293/validator/gates.json create mode 100644 evidence/h3-ref2va-lossless-293/validator/integrity.py create mode 100644 evidence/h3-ref2va-lossless-293/validator/smoke.py create mode 100644 evidence/h3-ref2va-lossless-293/validator/sources.json create mode 100644 evidence/h3-ref2va-lossless-293/validator/upstream_metrics.py create mode 100644 evidence/h3-ref2va-lossless-293/validator/weights.json create mode 100644 tests/test_h3_lossless_validator.py diff --git a/evidence/h3-ref2va-lossless-293/README.md b/evidence/h3-ref2va-lossless-293/README.md new file mode 100644 index 000000000..fdaca941f --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/README.md @@ -0,0 +1,73 @@ +# MiniMax-H3 Ref2VA lossless follow-up + +Status: **INCONCLUSIVE**. No new lossless performance gain is qualified. + +The requester requires lossless acceleration, no audio-quality degradation, and +both 5-second and 15-second outputs. The accepted scope is recorded in +[the campaign update](https://github.com/InferMatrixLab/InferRSI/issues/293#issuecomment-6573035907). +The old Cache-DiT result is lossy and cannot satisfy this request. The historical +`h3-ref2va-ascend-293` bundle is preserved byte for byte. + +## Current delivery + +The new validator rejects two equally truncated audio tracks, checks audio +timestamps and decoded sample coverage, and authenticates the actual bytes used +for both LPIPS networks. Weight loading is explicit and offline, with strict +parameter loading; the complete scorer state was compared against LPIPS 0.1.4's +previous loader. The synthetic regression is not model-quality evidence. + +Old perceptual gates remain diagnostic. A media-threshold PASS does not become a +lossless PASS: this validator returns INCONCLUSIVE for unequal decoded outputs +that pass those thresholds. It records audio and video equality separately. +Control repetitions are themselves non-bitwise-identical; any tolerance-based +adjudication needs a separately frozen matched repeatability protocol. This +bundle does not certify subjective sound quality, meaning, or lip synchronization. + +## Experiment plan + +- Native BF16, dense attention, 50 denoising steps, same references/seeds and + resolution. Disable approximate block caches, reference-KV reuse, quantization + and sparse attention in every arm. +- First screen exact weight placement: AllGather, rank-local streaming, and a + 16-layer resident prefix. The retained [screening plan](plan/screening.json) + fixes one source for all three arms. This is an existing-configuration search. +- Screen 5 seconds first with two warmups and one measured request per server, + inside one eight-card reservation. A pilot is not a repeated E2E qualification. +- After a candidate passes quality, run fresh A/B/B/A servers at **both 5 and + 15 seconds**, two warmups and three measured requests per duration/server. + Freeze the selected final source/configuration and quality protocol first. +- Report HTTP upload-to-full-MP4 E2E, independent server aggregates, completion, + sampled HBM and actual worker exit codes. Run stage profiling separately from + unprofiled timings. Do not estimate p95/p99 from the small sample set. +- Compare the final combination directly against the original dense baseline; + keep the matched diagnostic baseline shutdown budget explicit. Do not add + separate optimization percentages or claim a global optimum. + +## Sources and attribution + +| Item | Source | Current evidence | +| --- | --- | --- | +| Exact weight placement | [vLLM-Omni #5929](https://github.com/vllm-project/vllm-omni/pull/5929), component-selective DLO; current control source in the plan | Existing implementation; 16-layer choice is a campaign hypothesis, no gain claimed | +| HTTP workload and video/audio metrics | Prior campaign's frozen vLLM-Omni accuracy/quantization-test extracts | Metrics retained, old permissive audio gate is not a no-degradation certificate | +| Audio coverage repair | [review counterexample](https://github.com/InferMatrixLab/InferRSI/pull/294#discussion_r4432729023) | Reproduced with synthetic equal truncated tracks; new validator rejects | +| Authenticated scorer | [review finding](https://github.com/InferMatrixLab/InferRSI/pull/294#discussion_r4432729025), installed LPIPS 0.1.4 `LPIPS` and `alexnet` loaders | Explicit verified-byte loads; complete state equality checked on CPU | +| No approximation as an acceptance constraint | Requester's follow-up, linked above | Source/configuration audit plus new output validation still required for each promoted optimization | + +## Reproduction and limits + +The optional CPU scorer needs NumPy, PyTorch, torchvision, LPIPS 0.1.4 and FFmpeg. +Supply the two already provisioned weight files explicitly; the hashes are in +`validator/weights.json`. The validator does not download weights. + +```bash +python validator/compare.py --baseline --candidate \ + --frames 124 --alexnet-weights --lpips-weights \ + --output +``` + +Use 362 frames for the existing 15-second request contract. The 2048-sample upper +padding allowance is two AAC frames, not permission to truncate the expected +audio duration. The process returns 0 for exact decoded parity, 1 for a failed +media gate, and 2 for nonidentical outputs requiring further adjudication; +execution errors do not establish a campaign verdict. Model files, media and +host scripts/logs remain external. No completed 5/15-second E2E result is claimed. diff --git a/evidence/h3-ref2va-lossless-293/patch/README.md b/evidence/h3-ref2va-lossless-293/patch/README.md new file mode 100644 index 000000000..abd1d830b --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/patch/README.md @@ -0,0 +1,11 @@ +# Engine delivery + +All accepted engine changes remain consolidated in +[vLLM-Omni #9698](https://github.com/vllm-project/vllm-omni/pull/9698). +No new engine patch is accepted by this preparation slice. + +Initial configuration screening uses runtime +`098b51f6b96dd160a0959e74e8822422527ff06d`, which is runtime-identical to the +documentation-only PR head `922f1e9f1cb85b9f2b66d86e1eb500e6872b2dcb`. +The original dense baseline for final comparison remains +`c7f56f0155953c7d4a77276a2f4fcbd844ab0da2`. diff --git a/evidence/h3-ref2va-lossless-293/plan/integrity.json b/evidence/h3-ref2va-lossless-293/plan/integrity.json new file mode 100644 index 000000000..868b134b5 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/plan/integrity.json @@ -0,0 +1,11 @@ +{ + "validator/compare.py": "af3d7e05763e4b80aaaceecf771a76d94d810a6b134c14ec1a8537a4a5fde6f0", + "validator/integrity.py": "3ddaadcaf04b7e8962b5397454d52d5f26c0c7c8933505bb83f3c12e1fdcae10", + "validator/smoke.py": "54299a636b6f59b337b66706469e17c6fbfa7412adf918f1e626a3a84e6543a2", + "validator/upstream_metrics.py": "d76f3c66300e4a396fedb92da7a24e436f6241102c65032eb1eb16593cdabf71", + "validator/gates.json": "aa4330075b6da7ab268e6ef092c42b0e037681695b54b238b09692c21d58bb9a", + "validator/sources.json": "9ad2c43f3943be23944e5b5fcd1677d66e3f4a1b66c32eba3bf50dfb4921ad01", + "validator/weights.json": "605e463a0b64244974f5475dead6b3bdc9cc4899930801e161cdf48d8c057171", + "plan/screening.json": "689fb482e8d2a1648f7a3efead9028916da0d2237870e64f8beca95119a7b1d2", + "results/validator-regression.json": "7545936f2f1835c0eed427e822b8007caaddf1076e14249bdcb301d71518974d" +} diff --git a/evidence/h3-ref2va-lossless-293/plan/screening.json b/evidence/h3-ref2va-lossless-293/plan/screening.json new file mode 100644 index 000000000..cc5ac3860 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/plan/screening.json @@ -0,0 +1,53 @@ +{ + "status": "frozen_screening_not_acceptance", + "issue": "https://github.com/InferMatrixLab/InferRSI/issues/293#issuecomment-6573035907", + "source_revision": "098b51f6b96dd160a0959e74e8822422527ff06d", + "image_id": "sha256:7fc6c73bd19a6ef4ee8d6048009972234796a1fc1bd3067168a425343cc476f7", + "duration_s": 5, + "steps": 50, + "seed": 0, + "warmups": 2, + "measured_requests_per_server": 1, + "profiles": { + "allgather": { + "quality": "lossless", + "serve_args": [ + "--diffusion-attention-backend", + "FLASH_ATTN" + ] + }, + "ranklocal0": { + "quality": "lossless", + "serve_args": [ + "--diffusion-attention-backend", + "FLASH_ATTN", + "--diffusion-offload-config", + "{\"mode\": \"layer\", \"components\": [\"dit\"], \"layer_options\": {\"dit\": {\"weight_transfer\": \"rank-local\", \"resident_layers\": 0}}}" + ] + }, + "ranklocal16": { + "quality": "lossless", + "serve_args": [ + "--diffusion-attention-backend", + "FLASH_ATTN", + "--diffusion-offload-config", + "{\"mode\": \"layer\", \"components\": [\"dit\"], \"layer_options\": {\"dit\": {\"weight_transfer\": \"rank-local\", \"resident_layers\": 16}}}" + ] + } + }, + "source": "vLLM-Omni PR5929 component-selective DLO, ea3cf9989; campaign-authored 16-layer hypothesis", + "stop_policy": "Stop on first failed arm. No final acceptance from this screen. Final AB/BA at 5s and 15s required.", + "reservation": "one 8-card reservation, 6h hard, 315m soft, 100m per arm; 1200s HTTP and execution deadlines", + "quality": "No approximate cache, quantization, sparse attention, step reduction or reference-KV reuse. New coverage/authenticated-weight validator; nonidentical media remains INCONCLUSIVE pending matched repeatability adjudication.", + "script_sha256": { + "run_host.sh": "805c2b2cdf22b9ac85745c9755bf4b1bfa96f13a00e31125e1173d321affa30f", + "run_inner.sh": "d3a7ecd7108474a79e17783206e9b13fd85c8a6673d16751ed25cc9a435c9cf0", + "run_client.py": "7a3e697077524c9729ebdae78a533feb55777718b0db72ac3735e5164444bcde", + "run_heldout.py": "489e3048120fa34a1c8d089053f45f1d4cda4fd6fdf1652457d079302f77eded", + "allgather.json": "5b4ffd4364a6324811b2ad3a24eb623b9d9eb85093da2e025008f37267d4eaab", + "ranklocal0.json": "a2d01ea417740660f9794823f9d934662046b9546a4859322dae173383d16baf", + "ranklocal16.json": "4ec8b91f3f7b1714cae759e39e1507e92afb81a46203ce7a3f2b5282d30c79f5", + "campaign.sh": "2baf792b332a95fa55f90cd5c64ece0db565b7548ccb3e898f6d941209458929", + "launch.sh": "980943b33d94fa844d443d4811d234aa27064f05d3ec04dd6374c3401e18ef45" + } +} diff --git a/evidence/h3-ref2va-lossless-293/results/validator-regression.json b/evidence/h3-ref2va-lossless-293/results/validator-regression.json new file mode 100644 index 000000000..1b63c0610 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/results/validator-regression.json @@ -0,0 +1,51 @@ +{ + "synthetic": true, + "scope": "CPU validator regression only; not model quality or speedup", + "scorer_state_matches_legacy": true, + "weights": { + "alexnet_sha256": "7be5be791159472b1fbf3c69796f7cb30dca7ad8466c2df70058c37116cdee02", + "lpips_sha256": "df73285e35b22355a2df87cdb6b70b343713b667eddbda73e1977e0c860835c0" + }, + "complete_identity": { + "verdict": "PASS", + "media_gate_verdict": "PASS", + "scope": "decoded media equality; unequal outputs require separate baseline-repeatability adjudication", + "exact_video": true, + "exact_audio": true, + "coverage": [ + { + "expected_duration_s": 0.5, + "decoded_audio_end_s": 0.512 + }, + { + "expected_duration_s": 0.5, + "decoded_audio_end_s": 0.512 + } + ], + "scorer_identity": { + "alexnet_sha256": "7be5be791159472b1fbf3c69796f7cb30dca7ad8466c2df70058c37116cdee02", + "lpips_sha256": "df73285e35b22355a2df87cdb6b70b343713b667eddbda73e1977e0c860835c0" + }, + "metrics": { + "ssim": 1.0, + "psnr_db": "inf", + "lpips": 0.0, + "audio_spectral_cosine": 1.0000017285346985, + "audio_rms_ratio": 1.0, + "audio_samples": 16384 + } + }, + "legacy_truncated_identity": { + "verdict": "PASS", + "scope": "matched decoded video/audio parity only; not semantic or temporal quality", + "metrics": { + "ssim": 1.0, + "psnr_db": "inf", + "lpips": 0.0, + "audio_spectral_cosine": 1.0000033179918926, + "audio_rms_ratio": 1.0, + "audio_samples": 4096 + } + }, + "new_truncated_rejection": "Audio stream ends before the expected video duration" +} diff --git a/evidence/h3-ref2va-lossless-293/validator/compare.py b/evidence/h3-ref2va-lossless-293/validator/compare.py new file mode 100644 index 000000000..d3ac2d1ea --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/compare.py @@ -0,0 +1,230 @@ +"""Frozen campaign media gate; CPU only, separate from latency measurement. + +SSIM/PSNR use the upstream accuracy helper's FFmpeg filter definitions. +LPIPS and audio metrics are frozen verbatim in upstream_metrics.py. +""" + +import argparse +import hashlib +import json +import math +import re +import subprocess +import tempfile +from pathlib import Path + +import numpy as np +from integrity import audio_coverage, check_pcm_samples, load_scorer +from upstream_metrics import _audio_quality_metrics + + +def run(command): + return subprocess.run(command, check=True, capture_output=True, timeout=3600) + + +def probe(path, gates, frames): + result = json.loads( + run( + ["ffprobe", "-v", "error", "-count_frames", "-show_streams", "-of", "json", str(path)] + ).stdout + ) + streams = result["streams"] + video = [s for s in streams if s["codec_type"] == "video"] + audio = [s for s in streams if s["codec_type"] == "audio"] + if len(video) != 1 or len(audio) != 1: + raise ValueError("Exactly one video and audio stream required") + v, a = video[0], audio[0] + numerator, denominator = map(int, v["avg_frame_rate"].split("/")) + actual = ( + v["width"], + v["height"], + numerator / denominator, + int(v["nb_read_frames"]), + int(a["sample_rate"]), + a["channels"], + ) + expected = ( + gates["width"], + gates["height"], + gates["fps"], + frames, + gates["sample_rate"], + gates["channels"], + ) + if actual != expected: + raise ValueError(f"Media contract mismatch: {actual} != {expected}") + if abs(float(v["start_time"])) > 1 / gates["sample_rate"]: + raise ValueError("Video does not start at zero") + if abs(float(v["duration"]) - frames / gates["fps"]) > 1 / gates["sample_rate"]: + raise ValueError("Video timestamp duration differs from expected frame duration") + audio_start, audio_duration = float(a["start_time"]), float(a["duration"]) + if not math.isfinite(audio_start) or not math.isfinite(audio_duration): + raise ValueError("Invalid audio stream timestamps") + if abs(audio_start) > 1 / gates["sample_rate"]: + raise ValueError("Audio stream does not start at zero") + expected_duration = frames / gates["fps"] + if audio_duration < expected_duration - 1 / gates["sample_rate"]: + raise ValueError("Audio stream ends before the expected video duration") + max_duration = expected_duration + gates["max_audio_padding_samples"] / gates["sample_rate"] + if audio_duration > max_duration: + raise ValueError("Audio stream exceeds the allowed codec padding") + return audio_coverage(path, gates, frames) + + +def compare(baseline, candidate, gates, frames, alexnet_path, lpips_path, weights): + coverage = [probe(path, gates, frames) for path in (baseline, candidate)] + loss_fn, scorer_identity = load_scorer(alexnet_path, lpips_path, weights) + metrics = {} + for name, pattern in [ + ("ssim", r"All:([0-9.eE+-]+)"), + ("psnr_db", r"average:([0-9.eE+-]+|inf)"), + ]: + filter_name = "ssim" if name == "ssim" else "psnr" + output = run( + [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-i", + str(baseline), + "-i", + str(candidate), + "-lavfi", + f"[0:v][1:v]{filter_name}", + "-f", + "null", + "-", + ] + ).stderr.decode() + found = re.findall(pattern, output) + if not found: + raise ValueError(f"Missing {name} metric") + metrics[name] = float(found[-1]) + with tempfile.TemporaryDirectory() as directory: + videos, audios = [], [] + for index, path in enumerate((baseline, candidate)): + raw = Path(directory) / f"{index}.rgb" + run( + [ + "ffmpeg", + "-nostdin", + "-v", + "error", + "-i", + str(path), + "-map", + "0:v:0", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + str(raw), + ] + ) + video = np.memmap(raw, dtype=np.uint8, mode="r") + shape = (frames, gates["height"], gates["width"], 3) + if video.size != math.prod(shape): + raise ValueError("Decoded video length mismatch") + videos.append(video.reshape(shape)) + pcm = run( + [ + "ffmpeg", + "-nostdin", + "-v", + "error", + "-i", + str(path), + "-map", + "0:a:0", + "-f", + "f32le", + "-acodec", + "pcm_f32le", + "-", + ] + ).stdout + audio = np.frombuffer(pcm, dtype="= gates["min_ssim"] + and metrics["psnr_db"] >= gates["min_psnr_db"] + and metrics["lpips"] <= gates["max_lpips"] + and metrics["audio_spectral_cosine"] >= gates["min_audio_spectral_cosine"] + and gates["min_audio_rms_ratio"] + <= metrics["audio_rms_ratio"] + <= gates["max_audio_rms_ratio"] + ) + if math.isinf(metrics["psnr_db"]): + metrics["psnr_db"] = "inf" + return { + "verdict": "FAIL" + if not passed + else ("PASS" if exact_video and exact_audio else "INCONCLUSIVE"), + "media_gate_verdict": "PASS" if passed else "FAIL", + "scope": ( + "decoded media equality; unequal outputs require " + "separate baseline-repeatability adjudication" + ), + "exact_video": exact_video, + "exact_audio": exact_audio, + "coverage": coverage, + "scorer_identity": scorer_identity, + "metrics": metrics, + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--frames", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--alexnet-weights", type=Path, required=True) + parser.add_argument("--lpips-weights", type=Path, required=True) + args = parser.parse_args() + gates_path = Path(__file__).with_name("gates.json") + gates = json.loads(gates_path.read_text()) + weights = json.loads(Path(__file__).with_name("weights.json").read_text()) + result = compare( + args.baseline, + args.candidate, + gates, + args.frames, + args.alexnet_weights, + args.lpips_weights, + weights, + ) + result["inputs"] = { + name: hashlib.sha256(path.read_bytes()).hexdigest() + for name, path in [ + ("baseline", args.baseline), + ("candidate", args.candidate), + ("gates", gates_path), + ] + } + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + raise SystemExit({"PASS": 0, "FAIL": 1, "INCONCLUSIVE": 2}[result["verdict"]]) diff --git a/evidence/h3-ref2va-lossless-293/validator/gates.json b/evidence/h3-ref2va-lossless-293/validator/gates.json new file mode 100644 index 000000000..431b8a10d --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/gates.json @@ -0,0 +1,14 @@ +{ + "width": 1344, + "height": 768, + "fps": 24, + "sample_rate": 32000, + "channels": 2, + "max_audio_padding_samples": 2048, + "min_ssim": 0.97, + "min_psnr_db": 34.0, + "max_lpips": 0.2, + "min_audio_spectral_cosine": 0.8, + "min_audio_rms_ratio": 0.5, + "max_audio_rms_ratio": 2.0 +} diff --git a/evidence/h3-ref2va-lossless-293/validator/integrity.py b/evidence/h3-ref2va-lossless-293/validator/integrity.py new file mode 100644 index 000000000..17503e122 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/integrity.py @@ -0,0 +1,100 @@ +"""Coverage and explicit scorer loading for the lossless H3 follow-up. + +The historical campaign validator remains frozen. This version never downloads +weights and authenticates the same in-memory bytes that torch.load consumes. +""" + +import hashlib +import io +import json +import math +import subprocess +from pathlib import Path + + +def checked_bytes(path, expected): + data = Path(path).read_bytes() + actual = hashlib.sha256(data).hexdigest() + if actual != expected: + raise ValueError(f"Scorer weight hash mismatch: {actual} != {expected}") + return data + + +def load_scorer(alexnet_path, lpips_path, weights): + # Authenticate before imports/model construction; no download fallback. + trunk = checked_bytes(alexnet_path, weights["alexnet_sha256"]) + linear = checked_bytes(lpips_path, weights["lpips_sha256"]) + import lpips + import torch + + scorer = lpips.LPIPS(net="alex", pnet_rand=True, pretrained=False, verbose=False) + trunk_state = torch.load(io.BytesIO(trunk), map_location="cpu", weights_only=True) + features = {} + for name in scorer.net.state_dict(): + _, index, parameter = name.split(".") + features[name] = trunk_state[f"features.{index}.{parameter}"] + scorer.net.load_state_dict(features, strict=True) + linear_state = torch.load(io.BytesIO(linear), map_location="cpu", weights_only=True) + expected = {f"lin{index}.model.1.weight" for index in range(5)} + if set(linear_state) != expected: + raise ValueError("Unexpected LPIPS calibration keys") + for index in range(5): + getattr(scorer, f"lin{index}").load_state_dict( + {"model.1.weight": linear_state[f"lin{index}.model.1.weight"]}, strict=True + ) + scorer.eval() + return scorer, dict(weights) + + +def check_timeline(frames, expected_duration, rate, padding_samples): + """Reject missing heads/tails, gaps, overlaps, and non-finite timestamps.""" + tolerance = 1.1 / rate # one sample plus decimal timestamp serialization + padding = padding_samples / rate + end = 0.0 + for frame in frames: + start = float(frame["best_effort_timestamp_time"]) + duration = int(frame["nb_samples"]) / rate + if not math.isfinite(start) or duration <= 0: + raise ValueError("Invalid decoded audio timestamp/duration") + if abs(start - end) > tolerance: + raise ValueError("Audio has a missing head, timestamp gap, or overlap") + end = start + duration + if not frames or end < expected_duration - tolerance or end > expected_duration + padding: + raise ValueError("Audio does not cover the expected video duration") + return end + + +def audio_coverage(path, gates, frames): + duration = frames / gates["fps"] + probe = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "a:0", + "-show_frames", + "-show_entries", + "frame=best_effort_timestamp_time,nb_samples", + "-of", + "json", + str(path), + ], + check=True, + capture_output=True, + timeout=3600, + ) + audio_frames = json.loads(probe.stdout)["frames"] + end = check_timeline( + audio_frames, duration, gates["sample_rate"], gates["max_audio_padding_samples"] + ) + return {"expected_duration_s": duration, "decoded_audio_end_s": end} + + +def check_pcm_samples(samples, gates, frames): + expected = frames * gates["sample_rate"] / gates["fps"] + if ( + samples < math.floor(expected) + or samples > math.ceil(expected) + gates["max_audio_padding_samples"] + ): + raise ValueError("Decoded PCM does not cover the expected video duration") diff --git a/evidence/h3-ref2va-lossless-293/validator/smoke.py b/evidence/h3-ref2va-lossless-293/validator/smoke.py new file mode 100644 index 000000000..35453bb6e --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/smoke.py @@ -0,0 +1,90 @@ +import argparse +import importlib.util +import json +import subprocess +import tempfile +from pathlib import Path + +import compare +import lpips +import torch +from integrity import load_scorer + +parser = argparse.ArgumentParser( + description="Synthetic CPU validator regression, not model qualification" +) +parser.add_argument("--alexnet-weights", type=Path, required=True) +parser.add_argument("--lpips-weights", type=Path, required=True) +parser.add_argument("--output", type=Path, required=True) +args = parser.parse_args() +validator = Path(__file__).resolve().parent + +weights = json.loads((validator / "weights.json").read_text()) +alex = args.alexnet_weights +linear = args.lpips_weights +scorer, identity = load_scorer(alex, linear, weights) +original = lpips.LPIPS(net="alex", verbose=False).eval() +assert set(scorer.state_dict()) == set(original.state_dict()) +for name, tensor in scorer.state_dict().items(): + assert torch.equal(tensor, original.state_dict()[name]), name +gates = json.loads((validator / "gates.json").read_text()) +gates.update(width=64, height=64) +spec = importlib.util.spec_from_file_location( + "legacy_compare", validator.parent.parent / "h3-ref2va-ascend-293/validator/compare.py" +) +legacy = importlib.util.module_from_spec(spec) +spec.loader.exec_module(legacy) +with tempfile.TemporaryDirectory() as d: + p = Path(d) + for label, duration in [("complete", 0.5), ("truncated", 0.1)]: + subprocess.run( + [ + "ffmpeg", + "-nostdin", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:rate=24:duration=0.5", + "-f", + "lavfi", + "-i", + f"sine=frequency=440:sample_rate=32000:duration={duration}", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-ar", + "32000", + "-ac", + "2", + str(p / f"{label}.mp4"), + ], + check=True, + ) + complete = compare.compare( + p / "complete.mp4", p / "complete.mp4", gates, 12, alex, linear, weights + ) + assert complete["verdict"] == "PASS", complete + old = legacy.compare(p / "truncated.mp4", p / "truncated.mp4", gates, 12) + assert old["verdict"] == "PASS", old + try: + compare.compare(p / "truncated.mp4", p / "truncated.mp4", gates, 12, alex, linear, weights) + except ValueError as exc: + rejection = str(exc) + else: + raise AssertionError("truncated pair accepted") + result = { + "synthetic": True, + "scope": "CPU validator regression only; not model quality or speedup", + "scorer_state_matches_legacy": True, + "weights": identity, + "complete_identity": complete, + "legacy_truncated_identity": old, + "new_truncated_rejection": rejection, + } + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result)) diff --git a/evidence/h3-ref2va-lossless-293/validator/sources.json b/evidence/h3-ref2va-lossless-293/validator/sources.json new file mode 100644 index 000000000..cb03c722c --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/sources.json @@ -0,0 +1,17 @@ +[ + { + "path": "tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py", + "sha256": "8b9b0786434384fd10c07c7f6ad77c198db6b114b92f0d910a747eef031558b2", + "functions": [ + "_audio_quality_metrics", + "_canonical_audio" + ] + }, + { + "path": "benchmarks/diffusion/quantization_quality.py", + "sha256": "3fe9042491f71e953109854f19691da2cf186bc497d0a2b8228341f35c56972c", + "functions": [ + "compute_lpips_video" + ] + } +] diff --git a/evidence/h3-ref2va-lossless-293/validator/upstream_metrics.py b/evidence/h3-ref2va-lossless-293/validator/upstream_metrics.py new file mode 100644 index 000000000..96d718b94 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/upstream_metrics.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +# Frozen metric functions from vLLM-Omni c7f56f0155953c7d4a77276a2f4fcbd844ab0da2. +# Retained verbatim; campaign runner must enforce equal stream lengths first. +import numpy as np +import torch + + +def _canonical_audio(audio: object) -> np.ndarray: + """Return H3 audio as float32 [channels, samples].""" + array = np.asarray(audio, dtype=np.float32) + while array.ndim > 2 and array.shape[0] == 1: + array = array[0] + if array.ndim == 1: + array = array[None, :] + if array.ndim != 2: + raise ValueError(f"Expected mono/stereo audio, got shape {array.shape}") + if array.shape[0] > array.shape[1] and array.shape[1] <= 8: + array = array.T + if not 1 <= array.shape[0] <= 8: + raise ValueError(f"Expected channel-first audio, got shape {array.shape}") + if not np.isfinite(array).all(): + raise ValueError("Audio contains non-finite samples") + return np.ascontiguousarray(array) + +def _audio_quality_metrics(baseline: object, quantized: object) -> tuple[float, float]: + """Compare phase-tolerant spectral structure and overall signal energy.""" + baseline_array = _canonical_audio(baseline) + quantized_array = _canonical_audio(quantized) + if baseline_array.shape != quantized_array.shape: + raise ValueError( + "Audio shapes do not match for metric computation: " + f"baseline={baseline_array.shape}, quantized={quantized_array.shape}" + ) + + baseline_tensor = torch.from_numpy(baseline_array) + quantized_tensor = torch.from_numpy(quantized_array) + similarities: list[float] = [] + for n_fft in (512, 1024, 2048): + window = torch.hann_window(n_fft) + baseline_magnitude = torch.stft( + baseline_tensor, + n_fft=n_fft, + hop_length=n_fft // 4, + window=window, + return_complex=True, + ).abs() + quantized_magnitude = torch.stft( + quantized_tensor, + n_fft=n_fft, + hop_length=n_fft // 4, + window=window, + return_complex=True, + ).abs() + cosine = torch.nn.functional.cosine_similarity( + baseline_magnitude.flatten(1), + quantized_magnitude.flatten(1), + dim=1, + eps=1e-12, + ) + similarities.extend(float(value) for value in cosine) + + baseline_rms = float(torch.sqrt(torch.mean(baseline_tensor.square()))) + quantized_rms = float(torch.sqrt(torch.mean(quantized_tensor.square()))) + rms_ratio = quantized_rms / max(baseline_rms, 1e-12) + return float(np.mean(similarities)), rms_ratio + +def compute_lpips_video( + baseline_frames: np.ndarray, + quantized_frames: np.ndarray, + net: str = "alex", +) -> float: + """Compute mean per-frame LPIPS for a video pair. + + Args: + baseline_frames: (F, H, W, C) float array in [0, 1]. + quantized_frames: same shape. + + Returns: + Mean LPIPS across all frames. + """ + import lpips + + loss_fn = lpips.LPIPS(net=net).eval() + if torch.cuda.is_available(): + loss_fn = loss_fn.cuda() + + num_frames = min(len(baseline_frames), len(quantized_frames)) + scores = [] + for i in range(num_frames): + # Convert (H, W, C) float [0,1] -> (1, C, H, W) float [-1, 1] + f_bl = torch.from_numpy(baseline_frames[i]).permute(2, 0, 1).unsqueeze(0).float() * 2 - 1 + f_qt = torch.from_numpy(quantized_frames[i]).permute(2, 0, 1).unsqueeze(0).float() * 2 - 1 + if torch.cuda.is_available(): + f_bl, f_qt = f_bl.cuda(), f_qt.cuda() + with torch.no_grad(): + score = loss_fn(f_bl, f_qt).item() + scores.append(score) + return float(np.mean(scores)) diff --git a/evidence/h3-ref2va-lossless-293/validator/weights.json b/evidence/h3-ref2va-lossless-293/validator/weights.json new file mode 100644 index 000000000..977764e83 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/validator/weights.json @@ -0,0 +1,4 @@ +{ + "alexnet_sha256": "7be5be791159472b1fbf3c69796f7cb30dca7ad8466c2df70058c37116cdee02", + "lpips_sha256": "df73285e35b22355a2df87cdb6b70b343713b667eddbda73e1977e0c860835c0" +} diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index 661b42972..a17a51402 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -145,6 +145,6 @@ def test_checked_in_inventory(): repo = Path(__file__).resolve().parents[1] result = validate_inventory(repo) assert result["errors"] == [] - assert len(result["bundles"]) == 20 - assert result["checked_hashes"] == 425 + assert len(result["bundles"]) == 21 + assert result["checked_hashes"] == 434 assert result["unavailable"] # Historical raw artifacts are not in this checkout. diff --git a/tests/test_h3_lossless_validator.py b/tests/test_h3_lossless_validator.py new file mode 100644 index 000000000..3700f7379 --- /dev/null +++ b/tests/test_h3_lossless_validator.py @@ -0,0 +1,61 @@ +"""CPU regression checks for the new campaign's media integrity contract.""" + +import hashlib +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PATH = ROOT / "evidence/h3-ref2va-lossless-293/validator/integrity.py" +SPEC = importlib.util.spec_from_file_location("h3_lossless_integrity", PATH) +assert SPEC is not None and SPEC.loader is not None +integrity = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(integrity) + + +def audio_frame(start, samples=1024): + return {"best_effort_timestamp_time": str(start), "nb_samples": samples} + + +def test_accepts_complete_audio_with_codec_padding(): + frames = [audio_frame(i * 1024 / 32000) for i in range(16)] + assert integrity.check_timeline(frames, 0.5, 32000, 2048) == 0.512 + + +@pytest.mark.parametrize( + "frames", + [ + [], + [audio_frame(0)], # both arms could have the same truncated tail + [audio_frame(0.1)], + [audio_frame(0), audio_frame(0.064)], + [audio_frame(0), audio_frame(0.016)], + [audio_frame("nan")], + [audio_frame("inf")], + [audio_frame(0, 0)], + [audio_frame(0, 32000)], + ], +) +def test_rejects_incomplete_or_invalid_audio(frames): + with pytest.raises(ValueError): + integrity.check_timeline(frames, 0.5, 32000, 2048) + + +def test_pcm_coverage_checks_expected_length_not_just_pair_equality(): + gates = {"fps": 24, "sample_rate": 32000, "max_audio_padding_samples": 2048} + integrity.check_pcm_samples(165888, gates, 124) + for samples in (16000, 165888, 999999): + with pytest.raises(ValueError): + integrity.check_pcm_samples(samples, gates, 362) + + +def test_authenticates_actual_bytes_before_loading(tmp_path): + path = tmp_path / "synthetic-weights" + data = b"synthetic scorer data" + path.write_bytes(data) + digest = hashlib.sha256(data).hexdigest() + assert integrity.checked_bytes(path, digest) == data + path.write_bytes(b"different scorer data") + with pytest.raises(ValueError, match="hash mismatch"): + integrity.load_scorer(path, path, {"alexnet_sha256": digest, "lpips_sha256": digest}) diff --git a/tools/evidence_inventory.json b/tools/evidence_inventory.json index 023d17035..de019f72e 100644 --- a/tools/evidence_inventory.json +++ b/tools/evidence_inventory.json @@ -1,6 +1,11 @@ { "version": 1, "bundles": { + "h3-ref2va-lossless-293": { + "kind": "campaign", + "manifests": [{"path": "plan/integrity.json", "schema": "bundle-sha256"}], + "external": ["Model files, raw media, LPIPS weight files and host experiment logs are external. Weight digests are checked by the new scorer when explicitly supplied; inventory validation alone does not authenticate external weights or establish lossless quality."] + }, "h3-ref2va-ascend-293": { "kind": "campaign", "manifests": [ From e59b39a3ced3a18585576bcea8bda325699dde5b Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Mon, 2 Nov 2026 17:37:49 +0800 Subject: [PATCH 2/3] Record H3 screening failure and corrected offload provenance --- evidence/h3-ref2va-lossless-293/README.md | 23 ++- .../plan/candidate-sources-v2.json | 48 ++++++ .../plan/integrity-screening-v2.json | 6 + .../results/argv-preflight-v2.json | 65 ++++++++ .../results/offload-resolution-v2.json | 43 ++++++ .../results/screening-v1.json | 142 ++++++++++++++++++ tests/test_evidence_inventory.py | 2 +- tools/evidence_inventory.json | 5 +- 8 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 evidence/h3-ref2va-lossless-293/plan/candidate-sources-v2.json create mode 100644 evidence/h3-ref2va-lossless-293/plan/integrity-screening-v2.json create mode 100644 evidence/h3-ref2va-lossless-293/results/argv-preflight-v2.json create mode 100644 evidence/h3-ref2va-lossless-293/results/offload-resolution-v2.json create mode 100644 evidence/h3-ref2va-lossless-293/results/screening-v1.json diff --git a/evidence/h3-ref2va-lossless-293/README.md b/evidence/h3-ref2va-lossless-293/README.md index fdaca941f..ca8fbfc75 100644 --- a/evidence/h3-ref2va-lossless-293/README.md +++ b/evidence/h3-ref2va-lossless-293/README.md @@ -43,6 +43,27 @@ bundle does not certify subjective sound quality, meaning, or lip synchronizatio keep the matched diagnostic baseline shutdown budget explicit. Do not add separate optimization percentages or claim a global optimum. +## Screening update + +The [first screening attempt](results/screening-v1.json) completed one 5-second +baseline measurement at **400.943 seconds HTTP E2E**, after two warmups. All eight +workers exited with code 0. This single baseline is not a speedup measurement. +The rank-local arm failed before inference because the harness combined a unified +offload policy with a legacy offload flag; the third arm was not run. + +The replacement harness passed [actual shell argument assembly](results/argv-preflight-v2.json) +and the [real configuration resolver](results/offload-resolution-v2.json) on CPU. +The latter also shows that rank-local with zero resident layers selects the +`layer_wise` backend, while 16 resident layers selects `distributed_layer_wise`. +These are complete configuration candidates, not a pure collective comparison. +The corrected campaign must rerun the baseline and both candidates together. + +[Candidate provenance](plan/candidate-sources-v2.json) distinguishes existing +configuration mechanisms from the unqualified H3 NPU adaptation of QKV packing. +No candidate gain, 15-second result, or audio-quality acceptance is claimed. +Original frozen validator and screening files remain unchanged; these additions +have their own hash manifest. + ## Sources and attribution | Item | Source | Current evidence | @@ -70,4 +91,4 @@ padding allowance is two AAC frames, not permission to truncate the expected audio duration. The process returns 0 for exact decoded parity, 1 for a failed media gate, and 2 for nonidentical outputs requiring further adjudication; execution errors do not establish a campaign verdict. Model files, media and -host scripts/logs remain external. No completed 5/15-second E2E result is claimed. +host scripts/logs remain external. No completed paired 5/15-second E2E qualification is claimed. diff --git a/evidence/h3-ref2va-lossless-293/plan/candidate-sources-v2.json b/evidence/h3-ref2va-lossless-293/plan/candidate-sources-v2.json new file mode 100644 index 000000000..bd849a91d --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/plan/candidate-sources-v2.json @@ -0,0 +1,48 @@ +{ + "acceptance": "No lossy cache, no quantization, no omitted diffusion steps; complete audio and no demonstrated quality regression. No candidate has qualified yet.", + "candidates": [ + { + "name": "rank-local layer weight transfers", + "source": "https://github.com/vllm-project/vllm-omni/pull/5929", + "source_revision": "ea3cf9989", + "kind": "existing configuration, not new engine code", + "hypothesis": "Replicated host reads may avoid all-gather overhead", + "screening": "ranklocal0 versus allgather on the same reserved devices", + "status": "pending", + "gain": null, + "qualification_note": "Unified ranklocal0 policy selects layer_wise backend; ranklocal16 selects distributed_layer_wise. Report complete config gains, not isolated collective gains." + }, + { + "name": "bounded resident DiT layers", + "source": "https://github.com/vllm-project/vllm-omni/pull/5929", + "kind": "existing configuration, not new engine code", + "hypothesis": "Keep 16 layers resident to avoid repeated weight transfer if memory permits", + "screening": "ranklocal16 versus ranklocal0; combined versus allgather", + "status": "pending", + "gain": null, + "implementation_source": "https://github.com/vllm-project/vllm-omni/pull/5764", + "implementation_revision": "1c2a81f6d84aea4fff53bd2f894c2a287c237245", + "source_revision": "ea3cf99894f379c7732e6d62aa464a6f4285fbd9" + }, + { + "name": "combined strict QKV collective", + "source": "https://github.com/vllm-project/vllm-omni/pull/4150", + "source_revision": "5c0cdb7916bff945ba6b6a3c645a17e6973e5990", + "kind": "local NPU H3 adaptation of upstream proposal, reuses existing 5D collective", + "hypothesis": "One collective instead of three may reduce launch overhead, but packing and noncontiguous consumers can erase benefit", + "status": "10 CPU dispatch cases pass; real collective and NPU validation pending", + "gain": null + } + ], + "not_selected": [ + { + "source": "https://github.com/vllm-project/vllm-omni/pull/9353", + "reason": "Author reports eager regression; no basis for assuming benefit on current eager H3 workload." + }, + { + "source": "https://github.com/vllm-project/vllm-omni/pull/9510", + "reason": "Additional SHMEM NPU implementation/dependencies; draft is not evidence of qualified H3 lossless benefit." + } + ], + "reporting": "Report baseline, individual and combined E2E directly. Do not add component percentages. Synthetic collective speed is never reported as model E2E gain." +} diff --git a/evidence/h3-ref2va-lossless-293/plan/integrity-screening-v2.json b/evidence/h3-ref2va-lossless-293/plan/integrity-screening-v2.json new file mode 100644 index 000000000..fbead6f4e --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/plan/integrity-screening-v2.json @@ -0,0 +1,6 @@ +{ + "results/screening-v1.json": "0031fefa77b7992a3076d9310ed882e9b32798cd3ed9d22a974b9d6765e2c2c5", + "plan/candidate-sources-v2.json": "5089168edb63f5322ec698b8e9d038348c7eed4c4ca7fbcd5a6263be41988a2c", + "results/offload-resolution-v2.json": "a09a0eacecf5846c4bd0fabafc98aa6655631b469f56f63876b920f5837ecd40", + "results/argv-preflight-v2.json": "e166678cf7ef218962866b7db6279be4a1fe7609a22eea674e58b58fa55442f2" +} diff --git a/evidence/h3-ref2va-lossless-293/results/argv-preflight-v2.json b/evidence/h3-ref2va-lossless-293/results/argv-preflight-v2.json new file mode 100644 index 000000000..9d57154dc --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/results/argv-preflight-v2.json @@ -0,0 +1,65 @@ +{ + "pass": true, + "scope": "actual shell argv assembly, CPU only; not model startup qualification", + "arms": { + "allgather": [ + "--num-gpus", + "8", + "--usp", + "8", + "--ring", + "1", + "--text-encoder-tp-size", + "8", + "--vae-parallel-mode", + "tile", + "--vae-use-tiling", + "--vae-patch-parallel-size", + "8", + "--enable-distributed-layerwise-offload", + "--disable-log-stats", + "--diffusion-attention-backend", + "FLASH_ATTN" + ], + "ranklocal0": [ + "--num-gpus", + "8", + "--usp", + "8", + "--ring", + "1", + "--text-encoder-tp-size", + "8", + "--vae-parallel-mode", + "tile", + "--vae-use-tiling", + "--vae-patch-parallel-size", + "8", + "--disable-log-stats", + "--diffusion-attention-backend", + "FLASH_ATTN", + "--diffusion-offload-config", + "{\"mode\": \"layer\", \"components\": [\"dit\"], \"layer_options\": {\"dit\": {\"weight_transfer\": \"rank-local\", \"resident_layers\": 0}}}" + ], + "ranklocal16": [ + "--num-gpus", + "8", + "--usp", + "8", + "--ring", + "1", + "--text-encoder-tp-size", + "8", + "--vae-parallel-mode", + "tile", + "--vae-use-tiling", + "--vae-patch-parallel-size", + "8", + "--disable-log-stats", + "--diffusion-attention-backend", + "FLASH_ATTN", + "--diffusion-offload-config", + "{\"mode\": \"layer\", \"components\": [\"dit\"], \"layer_options\": {\"dit\": {\"weight_transfer\": \"rank-local\", \"resident_layers\": 16}}}" + ] + } +} diff --git a/evidence/h3-ref2va-lossless-293/results/offload-resolution-v2.json b/evidence/h3-ref2va-lossless-293/results/offload-resolution-v2.json new file mode 100644 index 000000000..021f4cf7a --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/results/offload-resolution-v2.json @@ -0,0 +1,43 @@ +{ + "source": "vllm_omni/diffusion/offloader/config.py at 098b51f6b96dd160a0959e74e8822422527ff06d", + "scope": "real pure config resolver, not engine startup", + "results": { + "allgather": { + "strategy": "distributed_layer_wise", + "components": [ + "dit" + ], + "transfers": { + "dit": "allgather", + "text_encoder": "rank-local" + }, + "resident_layers": 0, + "pin_memory": true + }, + "ranklocal0": { + "strategy": "layer_wise", + "components": [ + "dit" + ], + "transfers": { + "dit": "rank-local", + "text_encoder": "rank-local" + }, + "resident_layers": 0, + "pin_memory": true + }, + "ranklocal16": { + "strategy": "distributed_layer_wise", + "components": [ + "dit" + ], + "transfers": { + "dit": "rank-local", + "text_encoder": "rank-local" + }, + "resident_layers": 16, + "pin_memory": true + } + }, + "note": "ranklocal0 and ranklocal16 select different backend strategies automatically; treat as complete configuration candidates, not isolated communication microbenchmarks" +} diff --git a/evidence/h3-ref2va-lossless-293/results/screening-v1.json b/evidence/h3-ref2va-lossless-293/results/screening-v1.json new file mode 100644 index 000000000..45041ca90 --- /dev/null +++ b/evidence/h3-ref2va-lossless-293/results/screening-v1.json @@ -0,0 +1,142 @@ +{ + "verdict": "INCONCLUSIVE", + "scope": "one baseline screening sample; no paired speedup and no audio-quality acceptance", + "source_sha": "098b51f6b96dd160a0959e74e8822422527ff06d", + "image_digest": "sha256:7fc6c73bd19a6ef4ee8d6048009972234796a1fc1bd3067168a425343cc476f7", + "physical_devices": "0,1,2,3,4,5,6,7", + "samples": [ + { + "label": "warmup-5-0", + "e2e_ms": 547637.3106520623, + "http_status": 200, + "response_metrics": { + "X-Request-Id": "video_sync-b30931e63ed04785", + "X-Inference-Time-S": "547.532", + "X-Stage-Durations": "{\"queue_wait_ms\":0.4670824855566025,\"stage_0_gen_ms\":546236.5500926971}", + "X-Peak-Memory-MB": "19908.000" + } + }, + { + "label": "warmup-5-1", + "e2e_ms": 459427.02579218894, + "http_status": 200, + "response_metrics": { + "X-Request-Id": "video_sync-8a7cc6966a1e6806", + "X-Inference-Time-S": "459.365", + "X-Stage-Durations": "{\"queue_wait_ms\":1.1879904195666313,\"stage_0_gen_ms\":458093.4302806854}", + "X-Peak-Memory-MB": "19948.000" + } + }, + { + "label": "measured-5-seed0-0", + "e2e_ms": 400942.9486896843, + "http_status": 200, + "response_metrics": { + "X-Request-Id": "video_sync-ab09df02b6d7997f", + "X-Inference-Time-S": "400.882", + "X-Stage-Durations": "{\"queue_wait_ms\":0.40886085480451584,\"stage_0_gen_ms\":399828.3169269562}", + "X-Peak-Memory-MB": "19908.000" + } + } + ], + "worker_exit_receipt": [ + { + "name": "DiffusionWorker-0", + "pid": 60, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-1", + "pid": 61, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-2", + "pid": 62, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-3", + "pid": 63, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-4", + "pid": 64, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-5", + "pid": 65, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-6", + "pid": 66, + "exit_code": 0, + "alive": false + }, + { + "name": "DiffusionWorker-7", + "pid": 67, + "exit_code": 0, + "alive": false + } + ], + "baseline_service_exit": 0, + "baseline_container_exit": 0, + "ranklocal0": { + "status": "STARTUP_FAILURE", + "reason": "diffusion_offload_config cannot be combined with legacy enable_*_offload flags", + "service_exit": 1, + "container_exit": 1, + "inference_samples": 0 + }, + "ranklocal16": "NOT_RUN", + "job_exit": 1, + "external_artifacts": [ + { + "arm": "allgather", + "name": "service.log", + "bytes": 190298, + "sha256": "8f83809d1fe577eb188a942d3102faf7b84ec2418bb9820e8c9a2fbfb464e05b" + }, + { + "arm": "allgather", + "name": "runtime_settings.json", + "bytes": 1038, + "sha256": "011e702b39969659b02a76dd7c913080f57fd25b7780939728dbb21fa3bf437d" + }, + { + "arm": "allgather", + "name": "source_revision.txt", + "bytes": 41, + "sha256": "22d450d68401cb99248018c8b3a8c948f3797938720dbb799ff890d54a8d95cf" + }, + { + "arm": "ranklocal0", + "name": "service.log", + "bytes": 31582, + "sha256": "9a27256287efc201521996fb958698e73a973ad11efca5c39e81886722c02f5e" + }, + { + "arm": "ranklocal0", + "name": "runtime_settings.json", + "bytes": 1221, + "sha256": "61a23b9978ae0a8de0f6256fe3ed91854572f49cddb3472b3689710f78d095c5" + }, + { + "arm": "ranklocal0", + "name": "source_revision.txt", + "bytes": 41, + "sha256": "22d450d68401cb99248018c8b3a8c948f3797938720dbb799ff890d54a8d95cf" + } + ], + "lesson": "Assemble actual command arguments before reserving devices. A unified offload policy replaces the legacy flag; the engine correctly rejected conflicting configuration. Repeat all arms under a new reservation. A startup failure provides no speed or quality evidence." +} diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index a17a51402..21e9a511c 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -146,5 +146,5 @@ def test_checked_in_inventory(): result = validate_inventory(repo) assert result["errors"] == [] assert len(result["bundles"]) == 21 - assert result["checked_hashes"] == 434 + assert result["checked_hashes"] == 438 assert result["unavailable"] # Historical raw artifacts are not in this checkout. diff --git a/tools/evidence_inventory.json b/tools/evidence_inventory.json index de019f72e..038ee786b 100644 --- a/tools/evidence_inventory.json +++ b/tools/evidence_inventory.json @@ -3,7 +3,10 @@ "bundles": { "h3-ref2va-lossless-293": { "kind": "campaign", - "manifests": [{"path": "plan/integrity.json", "schema": "bundle-sha256"}], + "manifests": [ + {"path": "plan/integrity.json", "schema": "bundle-sha256"}, + {"path": "plan/integrity-screening-v2.json", "schema": "bundle-sha256"} + ], "external": ["Model files, raw media, LPIPS weight files and host experiment logs are external. Weight digests are checked by the new scorer when explicitly supplied; inventory validation alone does not authenticate external weights or establish lossless quality."] }, "h3-ref2va-ascend-293": { From f325d3c807c1e886e1e40cbc47ae16db9ba1eef0 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 4 Nov 2026 09:58:53 +0000 Subject: [PATCH 3/3] Pin the lossless bundle manifests instead of the shared inventory census discussion_r4440576147: concurrent registrations overwrite the same bundle and hash totals, and equal integers auto-merge to the wrong combined census. Keep full-tree validation, and check the two h3-ref2va-lossless-293 manifests and their 13 digests directly. Agent-assisted-by: Grok 4.7 (review follow-up) Signed-off-by: amy-why-3459 Co-authored-by: amy-why-3459 --- tests/test_evidence_inventory.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index e47204e81..98f7f1df2 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -6,7 +6,7 @@ import pytest -from tools.validate_evidence import SEGMENTS, validate_inventory +from tools.validate_evidence import SEGMENTS, validate_inventory, validate_manifest def write_json(path, value): @@ -145,6 +145,20 @@ def test_checked_in_inventory(): repo = Path(__file__).resolve().parents[1] result = validate_inventory(repo) assert result["errors"] == [] - assert len(result["bundles"]) == 22 - assert result["checked_hashes"] == 443 assert result["unavailable"] # Historical raw artifacts are not in this checkout. + # This bundle's manifests stay registered on their own. Another campaign can + # land without rewriting a shared bundle or hash total. + inventory = json.loads((repo / "tools/evidence_inventory.json").read_text(encoding="utf-8")) + entry = inventory["bundles"]["h3-ref2va-lossless-293"] + assert [(item["path"], item["schema"]) for item in entry["manifests"]] == [ + ("plan/integrity.json", "bundle-sha256"), + ("plan/integrity-screening-v2.json", "bundle-sha256"), + ] + bundle = repo / "evidence" / "h3-ref2va-lossless-293" + unavailable = [] + checked = sum( + validate_manifest(repo, bundle, manifest, unavailable) for manifest in entry["manifests"] + ) + assert unavailable == [] + assert checked == 13 # plan/integrity.json (9) and plan/integrity-screening-v2.json (4). + assert any(item["name"] == "h3-ref2va-lossless-293" for item in result["bundles"])