diff --git a/README.md b/README.md index 79b25c3..4401493 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,14 @@ - [Stage 3](STAGE3.md) — hard constraints, причины отбраковки кандидатов, materiality и cooldown. - [Stage 4](STAGE4.md) — hybrid chain, model blending и газовые теги как context-only сигналы. - [Stage 5](STAGE5.md) — uncertainty, applicability, robustness и policy guardrails без action model. +- [Stage 6](STAGE6.md) — acceptance pack, reproducible demo checks and final handoff limits. - [Code walkthrough](CODE_WALKTHROUGH.md) — папки, файлы и хронология вызовов почти построчно. - [ML system design](DESIGN.md#8-ml-неопределённость-и-модель-последствий) — обучение, метрики, анализ ошибок и жизненный цикл модели; общие контракты и данные описаны в том же документе. - [Материалы задания](materials/README.md) — ТЗ, схемы и исходные данные. ## Текущее состояние проекта -Реализация дошла до Stage 5 и содержит desktop UI. Часть команд и возможностей в +Реализация дошла до Stage 6 и содержит desktop UI. Часть команд и возможностей в дизайн-документе по-прежнему целевые; актуальный исполняемый контракт описан ниже. Сейчас реализованы: @@ -34,6 +35,8 @@ gas context для `ht:F9`, `ht:F22`, `ht:Q21` без включения реального управления газом. - stage 5: empirical upper estimate для прогноза серы, applicability/OOD gate, robustness reporting и materiality/cooldown policy helpers без включения action model. +- stage 6: приемочный контур `accept-stage6`, воспроизводимый прогон трех model-demo + сценариев, проверка journal-файлов и явная фиксация ограничений финальной демонстрации. Полноценной промышленной ML-модели и управления реальными уставками пока нет. Доступен локальный desktop UI на Python: он запускает существующие model-demo сценарии, показывает @@ -49,6 +52,10 @@ quality-agent сначала проверяет область применим upper sulfur. Missing/OOD/отсутствующий upper не превращаются в pass. Это не action model: backend по-прежнему не рекомендует реальные setpoint-изменения и не управляет газом. +Stage 6 упаковывает финальную приемку: команда `accept-stage6` валидирует конфигурацию, +прогоняет три model-demo сценария, проверяет ожидаемый `abstain` и наличие journal-файлов. +History artifact serving в CLI/UI остается отдельной следующей задачей. + ## Проверка Нужны совместимое с проектом Python-окружение, Git LFS и `tar` с поддержкой RAR. @@ -59,7 +66,9 @@ backend по-прежнему не рекомендует реальные setpo python -m venv .venv python -m pip install -r requirements.txt python -m source.main validate-stage0 +python -m source.main accept-stage6 python -m pytest +python -m pytest global_tests/test_stage6_acceptance.py python -m pytest global_tests/test_stage5_uncertainty_policy.py python -m ruff check . python -m ruff format --check . diff --git a/STAGE6.md b/STAGE6.md new file mode 100644 index 0000000..9de44f2 --- /dev/null +++ b/STAGE6.md @@ -0,0 +1,71 @@ +# Stage 6: acceptance and demonstration + +> Current status: Stage 6 is an acceptance pack. It does not add a new ML model, +> action model, real setpoint control, or history-artifact serving in the UI. + +## Purpose + +Stage 6 turns the implemented prototype into a reproducible handoff package. A +second participant should be able to create an environment, run the documented +checks, reproduce the model-demo scenarios, inspect journals, and understand the +remaining limits without reading the whole codebase first. + +## What Is Implemented + +- `python -m source.main accept-stage6` runs the final acceptance smoke. +- The command validates configs, contracts and fixtures through `validate_stage0()`. +- It runs `blend_normal`, `blend_risk` and `blend_missing` through the existing + deterministic model-demo cycle. +- It verifies the expected status for each scenario: all three remain `abstain`. +- It verifies that each run writes the required journal files: + `result.json`, `input.json`, `trace.jsonl` and `candidates.jsonl`. +- It prints a JSON summary with `passed`, `validation`, `scenarios`, + `environment`, `limitations` and `issues`. + +By default, acceptance journals are written under `runs/stage6/`. For tests or +clean demos, pass an explicit temporary directory: + +```bash +python -m source.main accept-stage6 --run-dir .test_tmp/stage6-runs +``` + +## Demonstration Flow + +Recommended clean-run sequence: + +```bash +python -m venv .venv +python -m pip install -r requirements.txt +python -m source.main validate-stage0 +python -m source.main accept-stage6 +python -m source.ui --smoke --scenario blend_risk +python -m pytest +``` + +Desktop UI remains available: + +```bash +python -m source.ui --scenario blend_risk +``` + +The UI shows the current model-demo capability: model blending diagnostics, +constraint checks, result export, and journal inspection. + +## Limits + +- Stage 6 does not connect a trusted Stage-5 history artifact to CLI or UI. +- Stage 6 does not create or retrain a model. +- Stage 6 does not recommend industrial setpoint changes. +- T95, cetane number and the complete product passport remain not assessed. +- `0.95` uncertainty coverage remains an empirical historical estimate, not an + industrial safety guarantee. + +## How To Check + +```bash +python -m pytest global_tests/test_stage6_acceptance.py +python -m pytest global_tests/test_ui.py global_tests/test_stage5_uncertainty_policy.py +python -m pytest +python -m ruff check . +python -m mypy source +``` diff --git a/global_tests/test_stage6_acceptance.py b/global_tests/test_stage6_acceptance.py new file mode 100644 index 0000000..94a2683 --- /dev/null +++ b/global_tests/test_stage6_acceptance.py @@ -0,0 +1,68 @@ +"""Stage-6 acceptance command tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from source.main import STAGE6_SCENARIOS, accept_stage6, main + + +def test_accept_stage6_cli_writes_journals_to_requested_run_dir( + tmp_path: Path, capsys: object +) -> None: + """Verify the public CLI performs the reproducible acceptance pass.""" + run_dir = tmp_path / "stage6-runs" + + exit_code = main(["accept-stage6", "--run-dir", str(run_dir)]) + + captured = capsys.readouterr() + payload = json.loads(captured.out) + assert exit_code == 0 + assert payload["passed"] is True + assert payload["validation"]["model_demo_fixtures"] == 3 + assert {item["scenario_id"] for item in payload["scenarios"]} == set(STAGE6_SCENARIOS) + for scenario in payload["scenarios"]: + assert scenario["status"] == "abstain" + assert scenario["passed"] is True + journal_dir = Path(scenario["journal_dir"]) + assert journal_dir.is_relative_to(run_dir) + for path in scenario["journal_files"].values(): + assert Path(path).is_file() + + +def test_accept_stage6_reports_failed_status_without_hiding_journal( + tmp_path: Path, +) -> None: + """Verify a status mismatch turns the acceptance result red.""" + expected = {scenario_id: "abstain" for scenario_id in STAGE6_SCENARIOS} + expected["blend_risk"] = "hold" + + payload = accept_stage6(run_dir=tmp_path, expected_statuses=expected) + + assert payload["passed"] is False + failed = [item for item in payload["scenarios"] if not item["passed"]] + assert [item["scenario_id"] for item in failed] == ["blend_risk"] + assert failed[0]["status"] == "abstain" + assert failed[0]["expected_status"] == "hold" + assert failed[0]["missing_journal_files"] == [] + assert any("blend_risk" in issue for issue in payload["issues"]) + + +def test_accept_stage6_summary_shape(tmp_path: Path) -> None: + """Verify the acceptance JSON exposes the fields used by README and demos.""" + payload = accept_stage6(run_dir=tmp_path) + + assert set(payload) == { + "passed", + "validation", + "scenarios", + "environment", + "limitations", + "issues", + } + assert isinstance(payload["validation"], dict) + assert isinstance(payload["scenarios"], list) + assert isinstance(payload["environment"]["python_version"], str) + assert isinstance(payload["environment"]["sklearn_version"], str) + assert payload["limitations"] diff --git a/source/main.py b/source/main.py index e8b0c94..309f533 100644 --- a/source/main.py +++ b/source/main.py @@ -4,16 +4,33 @@ import argparse import json +import platform import sys from datetime import datetime from pathlib import Path -from typing import Sequence +from typing import Mapping, Sequence from source.config import load_runtime_config, load_scenario, load_tag_dictionary -from source.contracts import DecisionContext, ProcessState, Recommendation +from source.contracts import DecisionContext, ProcessState, Recommendation, RecommendationStatus from source.data import build_state, load_prepared_dataset, prepare_dataset, write_prepared_dataset PROJECT_ROOT = Path(__file__).resolve().parent.parent +STAGE6_SCENARIOS: tuple[str, ...] = ("blend_normal", "blend_risk", "blend_missing") +STAGE6_EXPECTED_STATUSES: dict[str, str] = { + scenario_id: RecommendationStatus.ABSTAIN.value for scenario_id in STAGE6_SCENARIOS +} +STAGE6_JOURNAL_FILES: tuple[str, ...] = ( + "result.json", + "input.json", + "trace.jsonl", + "candidates.jsonl", +) +STAGE6_LIMITATIONS: tuple[str, ...] = ( + "Stage 6 is an acceptance and demonstration layer, not a new ML or action model.", + "Real setpoint recommendations remain disabled until a validated action model exists.", + "The desktop UI still demonstrates model-demo scenarios; " + "history artifact serving is next work.", +) def validate_stage0(root: Path = PROJECT_ROOT) -> dict[str, int]: @@ -40,7 +57,11 @@ def validate_stage0(root: Path = PROJECT_ROOT) -> dict[str, int]: } -def run_model_demo(scenario_id: str, root: Path = PROJECT_ROOT) -> Recommendation: +def run_model_demo( + scenario_id: str, + root: Path = PROJECT_ROOT, + run_dir: str | Path | None = None, +) -> Recommendation: """Run one deterministic stage-1 model-demo scenario on fixture data.""" from source.orchestrator import run_cycle @@ -57,7 +78,7 @@ def run_model_demo(scenario_id: str, root: Path = PROJECT_ROOT) -> Recommendatio scenario=scenario, config=config, context=DecisionContext(), - run_dir=root / config.runs_dir, + run_dir=_resolve_path(run_dir if run_dir is not None else config.runs_dir, root), ) @@ -118,6 +139,80 @@ def build_state_command( return build_state(data, as_of, scenario_config, config) +def accept_stage6( + root: Path = PROJECT_ROOT, + run_dir: str | Path | None = None, + expected_statuses: Mapping[str, str] = STAGE6_EXPECTED_STATUSES, +) -> dict[str, object]: + """Run the Stage-6 reproducibility and demonstration acceptance checks.""" + try: + import sklearn + + sklearn_version = sklearn.__version__ + except ImportError: + sklearn_version = "unavailable" + acceptance_run_dir = _resolve_path(run_dir if run_dir is not None else "runs/stage6", root) + issues: list[str] = [] + validation: dict[str, object] = {} + try: + validation.update(validate_stage0(root)) + except Exception as exc: + validation = {"error": str(exc)} + issues.append(f"validate_stage0 failed: {exc}") + + scenario_results: list[dict[str, object]] = [] + for scenario_id in STAGE6_SCENARIOS: + expected = expected_statuses[scenario_id] + try: + result = run_model_demo(scenario_id, root=root, run_dir=acceptance_run_dir) + journal_dir = acceptance_run_dir / result.run_id + missing = [name for name in STAGE6_JOURNAL_FILES if not (journal_dir / name).is_file()] + status = result.status.value + passed = status == expected and not missing + if status != expected: + issues.append(f"{scenario_id}: expected status {expected!r}, received {status!r}") + if missing: + issues.append(f"{scenario_id}: missing journal files {', '.join(missing)}") + scenario_results.append( + { + "scenario_id": scenario_id, + "expected_status": expected, + "status": status, + "passed": passed, + "run_id": result.run_id, + "journal_dir": journal_dir.as_posix(), + "journal_files": { + name: (journal_dir / name).as_posix() for name in STAGE6_JOURNAL_FILES + }, + "missing_journal_files": missing, + } + ) + except Exception as exc: + issues.append(f"{scenario_id}: {exc}") + scenario_results.append( + { + "scenario_id": scenario_id, + "expected_status": expected, + "status": None, + "passed": False, + "error": str(exc), + } + ) + + passed = not issues and all(bool(item.get("passed")) for item in scenario_results) + return { + "passed": passed, + "validation": validation, + "scenarios": scenario_results, + "environment": { + "python_version": platform.python_version(), + "sklearn_version": sklearn_version, + }, + "limitations": list(STAGE6_LIMITATIONS), + "issues": issues, + } + + def main(argv: Sequence[str] | None = None) -> int: """Parse the CLI command and run the requested backend command.""" if hasattr(sys.stdout, "reconfigure"): @@ -144,6 +239,14 @@ def main(argv: Sequence[str] | None = None) -> int: "--as-of", required=True, type=_parse_as_of, help="timezone-aware ISO datetime" ) state.add_argument("--config", default="config/runtime.toml", help="runtime config path") + accept = subparsers.add_parser( + "accept-stage6", help="run Stage-6 acceptance and demonstration checks" + ) + accept.add_argument( + "--run-dir", + default="runs/stage6", + help="directory for acceptance journals, relative to project root unless absolute", + ) args = parser.parse_args(argv) try: if args.command == "validate-stage0": @@ -162,6 +265,10 @@ def main(argv: Sequence[str] | None = None) -> int: state_result = build_state_command(args.dataset, args.scenario, args.as_of, args.config) print(state_result.model_dump_json(indent=2)) return 0 + if args.command == "accept-stage6": + acceptance_result = accept_stage6(run_dir=args.run_dir) + print(json.dumps(acceptance_result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if acceptance_result["passed"] else 1 except (FileNotFoundError, ValueError) as exc: print(f"error: {exc}", file=sys.stderr) return 1