From 043a230e3013b09f6633760e1cc0fde26c371883 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 19:17:56 +0800 Subject: [PATCH 01/65] docs: design pchronicle judge removal --- ...le-judge-removal-panic-hardening-design.md | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md new file mode 100644 index 00000000..6e1b0bbd --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md @@ -0,0 +1,133 @@ +# pChronicle Judge Removal and Panic Hardening Design + +## Goal + +Remove the complete pChronicle judgment vertical slice, eliminate production +`unwrap` and `expect` calls from the pChronicle library, and enforce strict +Clippy warnings in the active workspace. + +## Scope + +This change covers: + +- `persisting-pchronicle` judgment execution, persistence, aggregation, + protocol types, trajectory adapters, and public exports; +- `persisting-pchronicle-cli` judgment HTTP endpoints, Explorer projections, + and their tests; +- the twelve `clippy::unwrap_used` or `clippy::expect_used` findings compiled + by the default-feature `persisting-pchronicle` library target; +- the existing pVisor `clippy::type_complexity` finding that blocks workspace + `-D warnings`; +- local and CI Clippy commands for the active workspace. + +The standalone `persisting-dlcapt` component remains governed by its existing +strict workflow. Search and other subsystems excluded by `AGENTS.md` are not +enabled, modified, or included in the acceptance criteria. + +## Judge Removal Boundary + +The judgment capability is removed as one vertical slice rather than hidden +behind a feature flag or retained as a compatibility layer. + +The implementation will remove: + +- `judgment.rs`, `judge_service.rs`, and `judgment_summary.rs`; +- the trajectory `judge` and `judge_stats` adapters; +- judgment request, response, score, summary, scope, and method message types; +- judgment fields from ordinary trajectory statistics; +- public modules and re-exports for judgment behavior and storage; +- the `judgments.lance` path helper and judgment-specific layout exports; +- pChronicle's direct `reqwest` dependency; +- the CLI server's `/api/judgments` endpoint; +- judgment loading, aggregation, and presentation in Explorer responses; +- judgment-specific server tests and judgment-specific catalog fixture names. + +Existing `judgments.lance` directories on disk are left untouched. The new +code contains no judgment-specific discovery or interpretation and does not +read, write, report, or delete them. Generic catalog traversal may still see +an unknown derived Lance directory and must continue to exclude it from +canonical trajectory discovery. This avoids a destructive migration while +making the runtime and public API removal complete. + +This is an intentional breaking API change. No deprecated aliases, empty +response fields, or always-failing judgment entry points will remain. + +## Production Panic Hardening + +The production library target currently has twelve Clippy findings: ten +`expect` calls and two `unwrap` calls. Tests contain many more assertion-oriented +uses and are not part of this production-panic acceptance criterion. + +The findings will be removed by behavior-preserving control flow: + +- ACTF provenance and serialized-object assumptions become explicit errors; +- ACTF observation reference validation binds the present identifier directly; +- OpenAI corpus rows that violate the validated-object invariant return a + conversion error instead of panicking; +- revision JSON serialization is collected as a fallible operation; +- catalog single-plan selection uses an explicit checked branch; +- index-build admission returns a fallible result and callers propagate a + closed-semaphore error; +- a poisoned root-lock registry recovers the contained map instead of panicking. + +No `allow` or `expect` lint annotations will be introduced for these findings. +The target state is zero `clippy::unwrap_used` and zero +`clippy::expect_used` diagnostics for: + +```bash +cargo clippy -p persisting-pchronicle --lib --locked -- \ + -D clippy::unwrap_used -D clippy::expect_used +``` + +## Strict Clippy Policy + +The ordinary Rust lint command and the main CI lint job will treat every +Clippy warning as an error. The obsolete comments describing strict Clippy as +unsafe to run will be removed, and the compatibility target will delegate to +the strict command rather than maintaining a weaker path. + +The existing pVisor trajectory sink tuple will be replaced with a named type +alias or focused struct so `clippy::type_complexity` is fixed without a lint +suppression. + +A separate pChronicle production-panic lint target will run the command above. +It deliberately checks `--lib`, not `--all-targets`, so tests may continue to +use `unwrap` and `expect` as concise assertion helpers. Normal `-D warnings` +still applies to tests through the workspace all-target Clippy command. + +## Error and Compatibility Semantics + +Malformed external or persisted conversion input must return the existing +pChronicle error type or an `anyhow::Error` at the owning storage boundary. +Internal concurrency failures must be propagated where recovery is not safe. +Mutex poisoning in the process-local lock registry is recoverable because the +registry stores only weak lock references; recovering the map does not bypass +the per-dataset asynchronous lock or cross-process storage fencing. + +Removing the `judge` field from `TrajectoryStatsResponse` and removing +judgment HTTP response fields is intentionally not wire-compatible. Consumers +must stop requesting or decoding judgment data. + +## Verification + +The implementation is accepted when all of the following hold: + +1. No judgment modules, public types, routes, Explorer fields, or direct + pChronicle `reqwest` dependency remain. +2. Existing judgment files are not deleted by code or migration scripts. +3. The pChronicle library passes the production panic Clippy command. +4. pChronicle and pChronicle CLI targeted tests pass after obsolete judgment + tests are removed and affected stats/Explorer fixtures are updated. +5. The active workspace passes all-target Clippy with `-D warnings`, excluding + only `persisting-dlcapt`, which retains its separate strict workflow. +6. Rust formatting checks pass for all modified Rust files. + +## Non-Goals + +- Extracting judgment into a replacement crate. +- Preserving read-only access to historical judgment datasets. +- Deleting or migrating historical judgment data. +- Refactoring pChronicle's broader public facade or splitting its store, + formats, conversion, or query subsystems. +- Cleaning `unwrap` or `expect` calls in tests, Search, `persisting-dlcapt`, or + other crates. From c0a130a41a55978cca09f65acda3f9504763f9de Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 19:20:28 +0800 Subject: [PATCH 02/65] docs: include pchronicle web judge removal --- ...nicle-judge-removal-panic-hardening-design.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md index 6e1b0bbd..968f76f0 100644 --- a/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md +++ b/docs/superpowers/specs/2026-08-17-pchronicle-judge-removal-panic-hardening-design.md @@ -14,6 +14,10 @@ This change covers: protocol types, trajectory adapters, and public exports; - `persisting-pchronicle-cli` judgment HTTP endpoints, Explorer projections, and their tests; +- the standalone `pchronicle-web` judgment models, API client, analysis skill, + workspace UI, styling, and tests; +- current pChronicle architecture documentation that describes judgment as an + active product capability; - the twelve `clippy::unwrap_used` or `clippy::expect_used` findings compiled by the default-feature `persisting-pchronicle` library target; - the existing pVisor `clippy::type_complexity` finding that blocks workspace @@ -40,7 +44,9 @@ The implementation will remove: - pChronicle's direct `reqwest` dependency; - the CLI server's `/api/judgments` endpoint; - judgment loading, aggregation, and presentation in Explorer responses; -- judgment-specific server tests and judgment-specific catalog fixture names. +- judgment-specific server tests and judgment-specific catalog fixture names; +- the Web client's judgment fetch path, judgment-aware analysis context and + skill, score/verdict panels, and judgment-specific model fields. Existing `judgments.lance` directories on disk are left untouched. The new code contains no judgment-specific discovery or interpretation and does not @@ -118,9 +124,13 @@ The implementation is accepted when all of the following hold: 3. The pChronicle library passes the production panic Clippy command. 4. pChronicle and pChronicle CLI targeted tests pass after obsolete judgment tests are removed and affected stats/Explorer fixtures are updated. -5. The active workspace passes all-target Clippy with `-D warnings`, excluding +5. The standalone pChronicle Web tests and build pass with no judgment API or + model references. +6. Current architecture documentation no longer presents judgment as an + active pChronicle capability; historical data remains explicitly untouched. +7. The active workspace passes all-target Clippy with `-D warnings`, excluding only `persisting-dlcapt`, which retains its separate strict workflow. -6. Rust formatting checks pass for all modified Rust files. +8. Rust formatting checks pass for all modified Rust files. ## Non-Goals From 049265c76de89210740f2ab38c0087402182ecf9 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 19:23:46 +0800 Subject: [PATCH 03/65] docs: plan pchronicle judge removal --- ...chronicle-judge-removal-panic-hardening.md | 549 ++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-17-pchronicle-judge-removal-panic-hardening.md diff --git a/docs/superpowers/plans/2026-08-17-pchronicle-judge-removal-panic-hardening.md b/docs/superpowers/plans/2026-08-17-pchronicle-judge-removal-panic-hardening.md new file mode 100644 index 00000000..4a682950 --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-pchronicle-judge-removal-panic-hardening.md @@ -0,0 +1,549 @@ +# pChronicle Judge Removal and Panic Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove pChronicle's complete judgment vertical slice, eliminate production `unwrap` and `expect` calls from the pChronicle library, and make strict Clippy the default local and CI policy. + +**Architecture:** Remove the judgment capability at every owning and consuming boundary instead of feature-gating or deprecating it. Harden the remaining pChronicle production paths with explicit errors or infallible synchronization primitives, then enforce both workspace warning denial and a production-only panic lint. + +**Tech Stack:** Rust 2021, Cargo, Clippy, Tokio, Axum, DataFusion, Lance, Dioxus, mdBook, GitHub Actions, just. + +## Global Constraints + +- Existing `judgments.lance` directories must never be deleted or migrated. +- No compatibility stubs, deprecated judgment aliases, empty judgment fields, or always-failing judgment endpoints remain. +- Tests may retain assertion-oriented `unwrap` and `expect`; the production panic lint targets `persisting-pchronicle --lib` only. +- Search, TTAS, Queue, samplers, and `persisting-dlcapt` remain out of scope. +- `persisting-dlcapt` keeps its separate strict workflow and is excluded from the active-workspace Clippy command. +- No lint suppression is added for the twelve pChronicle production findings or pVisor's `type_complexity` finding. + +--- + +## File Map + +- `crates/persisting-pchronicle/src/{judgment.rs,judge_service.rs,judgment_summary.rs}`: delete judgment persistence, orchestration, and aggregation. +- `crates/persisting-pchronicle/src/operations/trajectory/{judge.rs,judge_stats.rs}`: delete typed judgment adapters. +- `crates/persisting-pchronicle/src/{lib.rs,messages.rs,layout/coords.rs,layout/mod.rs,operations/trajectory/mod.rs}`: remove public API, protocol, path, and stats integration. +- `crates/persisting-pchronicle/src/store/catalog/{discovery.rs,tests.rs}`: retain generic derived-Lance exclusion without judgment-specific names. +- `crates/persisting-pchronicle/Cargo.toml` and `Cargo.lock`: remove pChronicle's direct HTTP-client dependency. +- `crates/persisting-pchronicle-cli/src/server/{mod.rs,explorer.rs,tests.rs}` and `crates/persisting-pchronicle-cli/tests/server_http_contract.rs`: remove server and Explorer judgment contracts. +- `pchronicle-web/src/{model.rs,api.rs,agent.rs,workspace.rs,components.rs}` and `pchronicle-web/assets/workbench.css`: remove the Web judgment consumer and UI. +- `docs/src/pchronicle/**` and affected `docs/src/rfcs/*.md`: stop documenting judgment as an active capability. +- `crates/persisting-pchronicle/src/{convert/actf.rs,formats/actf.rs,formats/openai_corpus.rs,revision.rs}`: replace conversion and serialization panics. +- `crates/persisting-pchronicle/src/store/{index_build_gate.rs,root_write_lock.rs}` and `store/catalog/provider.rs`: replace synchronization and plan-selection panics. +- `crates/persisting-pvisor/src/cli/run.rs`: name the Chronicle sink tuple. +- `justfile` and `.github/workflows/ci.yml`: enforce strict Clippy and the pChronicle production panic lint. + +--- + +### Task 1: Remove the Backend Judgment Vertical Slice + +**Files:** +- Delete: `crates/persisting-pchronicle/src/judgment.rs` +- Delete: `crates/persisting-pchronicle/src/judge_service.rs` +- Delete: `crates/persisting-pchronicle/src/judgment_summary.rs` +- Delete: `crates/persisting-pchronicle/src/operations/trajectory/judge.rs` +- Delete: `crates/persisting-pchronicle/src/operations/trajectory/judge_stats.rs` +- Modify: `crates/persisting-pchronicle/src/lib.rs` +- Modify: `crates/persisting-pchronicle/src/messages.rs` +- Modify: `crates/persisting-pchronicle/src/layout/coords.rs` +- Modify: `crates/persisting-pchronicle/src/layout/mod.rs` +- Modify: `crates/persisting-pchronicle/src/operations/trajectory/mod.rs` +- Modify: `crates/persisting-pchronicle/src/store/catalog/discovery.rs` +- Modify: `crates/persisting-pchronicle/src/store/catalog/tests.rs` +- Modify: `crates/persisting-pchronicle/Cargo.toml` +- Modify: `Cargo.lock` +- Modify: `crates/persisting-pchronicle-cli/src/server/mod.rs` +- Modify: `crates/persisting-pchronicle-cli/src/server/explorer.rs` +- Modify: `crates/persisting-pchronicle-cli/src/server/tests.rs` +- Modify: `crates/persisting-pchronicle-cli/tests/server_http_contract.rs` + +**Interfaces:** +- Removes: all `Judge*`, `Judgment*`, `TrajectoryJudge*`, `SessionJudgeStats`, judgment path helpers, `judge_async`, `judge_stats_async`, and `/api/judgments`. +- Preserves: trajectory append, replay, stats, materialize, extract, revisions, catalog discovery, and generic derived-Lance filtering. + +- [ ] **Step 1: Replace the read-only judgment test with a failing removed-route contract** + +In `crates/persisting-pchronicle-cli/src/server/tests.rs`, replace the judgment read/write tests with this focused contract: + +```rust +#[tokio::test] +async fn removed_judgments_route_returns_not_found() { + use tower::ServiceExt; + + let root = json_dataset_root(); + let response = router(root.to_string_lossy().to_string()) + .oneshot( + axum::http::Request::builder() + .uri("/api/judgments?agent_id=model-json&session_id=json-session") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + std::fs::remove_dir_all(root).unwrap(); +} +``` + +- [ ] **Step 2: Run the route contract and verify RED** + +Run: + +```bash +cargo test -p persisting-pchronicle-cli removed_judgments_route_returns_not_found --locked +``` + +Expected: FAIL because the existing `/api/judgments` GET route returns a non-404 response. + +- [ ] **Step 3: Delete judgment ownership from pChronicle** + +Delete the five judgment implementation files. Remove their module declarations and re-exports from `lib.rs`; remove `judgment` from the crate-level ownership documentation. Remove the judgment types from `messages.rs`, including the `judge` field on `TrajectoryStatsResponse`. + +Make `stats_async` return the ordinary response directly: + +```rust +Ok(TrajectoryStatsResponse { + dataset: layers.event_log_path, + storage: request.storage, + agent_id: request.agent_id, + session_id: request.session_id, + row_count: layers.event_rows, + manifest_revision: RawEventLanceStore.stats(&session).await?.manifest_revision, + duplicate_event_ids, + status: if layers.event_rows > 0 { "ok" } else { "empty" }.into(), + note: format!( + "Canonical Lance event log: {} row(s){projection_note}", + layers.event_rows + ), +}) +``` + +Remove `StoryCoords::lance_judgment_path`, `story_lance_judgment_path`, and their test. Remove pChronicle's direct `reqwest` dependency from its manifest and let Cargo refresh the package dependency list in `Cargo.lock`. + +- [ ] **Step 4: Remove the CLI Server and Explorer contracts** + +Remove `JudgeRow` and `read_judge_rows` imports, the `/api/judgments` route, `session_judgments`, and the `judgments` handler. Change Explorer functions to consume only runs, turns, and events: + +```rust +pub(crate) fn run_page( + mut records: Vec, + query: &ExplorerRunsQuery, +) -> RunExplorerPage + +pub(crate) fn analyze( + run: RunSummary, + turns: &[StorylineTurn], + events: &[EventRecord], +) -> RunAnalysis + +pub(crate) fn turn_page( + turns: &[StorylineTurn], + events: &[EventRecord], + q: Option<&str>, + source: Option<&str>, + offset: usize, + limit: usize, +) -> ExplorerPage + +pub(crate) fn turn_detail( + item: &StorylineTurn, + events: &[EventRecord], +) -> TurnDetail +``` + +Remove judgment counts, average scores, verdicts, and per-turn judgment arrays from the corresponding serialized response structs and fixtures. Remove the obsolete judgment integration assertion in `server_http_contract.rs`. + +- [ ] **Step 5: Preserve generic derived-Lance discovery coverage** + +Rename the catalog fixture from `judgments.lance` to `derived-metrics.lance`, and generalize the discovery comment: + +```rust +// Derived Lance datasets are sidecars of a canonical Run, not trajectory +// sources. Never descend into their internal metadata and register it as an +// outer file source. +``` + +The test must still assert that only `trajectory.json` is registered. + +- [ ] **Step 6: Run backend formatting, references, and targeted tests** + +Run: + +```bash +cargo fmt --all -- --check +! git grep -n -E 'Judge(Row|Scope|Method|Sample|Score|Stats|Rubric|Dialogue|Trajectory)|Judgment|judge_(async|stats|trajectory)|judgments\.lance|story_lance_judgment_path' -- crates/persisting-pchronicle crates/persisting-pchronicle-cli +cargo test -p persisting-pchronicle --lib --locked +cargo test -p persisting-pchronicle-cli --lib --tests --locked +``` + +Expected: all commands pass; the source scan prints no matches. + +- [ ] **Step 7: Commit the backend removal** + +```bash +git add Cargo.lock crates/persisting-pchronicle crates/persisting-pchronicle-cli +git commit -m "refactor: remove pchronicle judgment capability" +``` + +--- + +### Task 2: Remove the pChronicle Web Judgment Consumer + +**Files:** +- Modify: `pchronicle-web/src/model.rs` +- Modify: `pchronicle-web/src/api.rs` +- Modify: `pchronicle-web/src/agent.rs` +- Modify: `pchronicle-web/src/workspace.rs` +- Modify: `pchronicle-web/src/components.rs` +- Modify: `pchronicle-web/assets/workbench.css` + +**Interfaces:** +- Removes: `Judgment`, `api::judgments`, `judgment_review`, judgment props/state, and score/verdict/rubric UI. +- Preserves: run analysis, turn inspection, read-only SQL evidence, failure/latency/tool/cohort skills, and LLM-assisted analysis. + +- [ ] **Step 1: Verify the Web source policy is RED** + +Run: + +```bash +! git grep -n -i -E 'judge|judgment' -- pchronicle-web/src pchronicle-web/assets +``` + +Expected: FAIL and print existing model, API, agent, workspace, component, CSS, and test references. + +- [ ] **Step 2: Remove judgment data from the Web model and API** + +Remove `Judgment`, judgment fields from `RunExplorerItem`, `RunAnalysis`, `TurnSummary`, and `TurnDetail`, and remove `api::judgments`. Keep all remaining wire fields unchanged. + +- [ ] **Step 3: Remove judgment-aware agent behavior** + +Remove `Judgment` imports, the `judgments` field from `AnswerRequest`, the `judgment_review` skill ID and match arm, and judgment arguments from `evidence_context` and `run_skill`. The remaining request shape is: + +```rust +pub struct AnswerRequest<'a> { + pub config: &'a LlmConfig, + pub user_message: &'a str, + pub run: &'a RunSummary, + pub analysis: &'a RunAnalysis, + pub turns: &'a [TurnSummary], + pub selected: Option<&'a TurnDetail>, + pub include_full_turn: bool, +} +``` + +Update agent tests so available skills and evidence assertions cover only retained sources. + +- [ ] **Step 4: Remove judgment state and presentation from the workspace** + +Remove judgment resource loading, refresh wiring, props, verdict/rubric helpers, score cards, per-turn judgment panels, and judgment-only tests. Remove CSS selectors that no retained component uses. Keep run overview, timeline, evidence, metrics, and Copilot layouts intact. + +- [ ] **Step 5: Verify the standalone Web package** + +Run: + +```bash +cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check +! git grep -n -i -E 'judge|judgment' -- pchronicle-web/src pchronicle-web/assets +cargo test --manifest-path pchronicle-web/Cargo.toml --locked +cargo check --manifest-path pchronicle-web/Cargo.toml --locked +``` + +Expected: all commands pass and the source scan is empty. + +- [ ] **Step 6: Commit the Web removal** + +```bash +git add pchronicle-web +git commit -m "refactor: remove judgment from pchronicle web" +``` + +--- + +### Task 3: Update Active pChronicle Documentation + +**Files:** +- Modify: `docs/src/pchronicle/concepts/facts-and-projections.md` +- Modify: `docs/src/pchronicle/concepts/facts-and-projections.zh.md` +- Modify: `docs/src/pchronicle/design/catalog.md` +- Modify: `docs/src/pchronicle/design/catalog.zh.md` +- Modify: `docs/src/pchronicle/design/trajectory-storage.md` +- Modify: `docs/src/pchronicle/design/trajectory-storage.zh.md` +- Modify: `docs/src/rfcs/0002-events-format.md` +- Modify: `docs/src/rfcs/0003-pchronicle-ownership.md` +- Modify: `docs/src/rfcs/0005-pchronicle-revision-lineage.md` + +**Interfaces:** +- Removes: documentation claims that pChronicle executes, persists, serves, or displays judgments. +- Preserves: the generic distinction between canonical facts, rebuildable projections, and revision lineage. + +- [ ] **Step 1: Verify the documentation policy is RED** + +Run: + +```bash +! git grep -n -i -E 'judge|judgment' -- docs/src +``` + +Expected: FAIL and list the current capability claims. + +- [ ] **Step 2: Remove active judgment claims without weakening generic architecture** + +Use neutral derived-data examples such as redaction, augmentation, enrichment, and export. In the ownership RFC remove the judgment persistence row and remove `judge` from the list of pChronicle workflows. In the revision RFC remove `judge` from the built-in kind examples while keeping extensible revision kinds. In the catalog docs describe the server as read-only without referring to a removed write API. + +- [ ] **Step 3: Verify docs and commit** + +Run: + +```bash +! git grep -n -i -E 'judge|judgment' -- docs/src +git diff --check -- docs/src +``` + +Expected: both commands pass. + +```bash +git add docs/src +git commit -m "docs: remove pchronicle judgment capability" +``` + +--- + +### Task 4: Eliminate pChronicle Production `unwrap` and `expect` + +**Files:** +- Modify: `crates/persisting-pchronicle/src/convert/actf.rs` +- Modify: `crates/persisting-pchronicle/src/formats/actf.rs` +- Modify: `crates/persisting-pchronicle/src/formats/openai_corpus.rs` +- Modify: `crates/persisting-pchronicle/src/revision.rs` +- Modify: `crates/persisting-pchronicle/src/store/catalog/provider.rs` +- Modify: `crates/persisting-pchronicle/src/store/index_build_gate.rs` +- Modify: `crates/persisting-pchronicle/src/store/root_write_lock.rs` + +**Interfaces:** +- Produces: zero `clippy::unwrap_used` and `clippy::expect_used` diagnostics for the default-feature pChronicle library. +- Preserves: existing conversion schemas, catalog plans, serialized revision representation, and single-index-build admission. + +- [ ] **Step 1: Run the production panic lint and verify RED** + +Run: + +```bash +cargo clippy -p persisting-pchronicle --lib --locked -- \ + -D clippy::unwrap_used -D clippy::expect_used +``` + +Expected: FAIL with findings in ACTF conversion, ACTF validation, OpenAI corpus conversion, revision serialization, catalog planning, index admission, and root-lock registration. + +- [ ] **Step 2: Replace conversion assumptions with explicit errors** + +Use `ok_or_else(...)?` for ACTF provenance and serialized-object access. Bind ACTF observation IDs with `if let Some(referenced_id)` instead of checking and then expecting. Convert an OpenAI corpus row with: + +```rust +let row = raw.as_object().ok_or_else(|| { + Error::Other(format!( + "OpenAI corpus {} row {} must be an object", + relative_path, ordinal + )) +})?; +``` + +Use existing error types and preserve the path/row context in every new message. + +- [ ] **Step 3: Make revision serialization fallible** + +Precompute both JSON string columns before `RecordBatch::try_new`: + +```rust +let parent_revision_ids = rows + .iter() + .map(|row| serde_json::to_string(&row.parent_revision_ids)) + .collect::>>()?; +let output_refs = rows + .iter() + .map(|row| serde_json::to_string(&row.output_refs)) + .collect::>>()?; +``` + +Pass those vectors to `StringArray::from` without any unwrap. + +- [ ] **Step 4: Remove storage-control panics** + +Select the single catalog plan with a checked error: + +```rust +1 => plans.pop().ok_or_else(|| { + DataFusionError::Internal("Catalog planned one source but produced no plan".into()) +})?, +``` + +Replace the index-build semaphore with a process-wide `tokio::sync::Mutex<()>`; `lock_owned().await` is infallible and exactly models single admission. Recover a poisoned root registry with: + +```rust +let mut locks = locks.lock().unwrap_or_else(std::sync::PoisonError::into_inner); +``` + +- [ ] **Step 5: Verify GREEN and run pChronicle tests** + +Run: + +```bash +cargo fmt --all -- --check +cargo clippy -p persisting-pchronicle --lib --locked -- \ + -D clippy::unwrap_used -D clippy::expect_used +cargo test -p persisting-pchronicle --lib --locked +``` + +Expected: all commands pass with zero production panic diagnostics. + +- [ ] **Step 6: Commit panic hardening** + +```bash +git add crates/persisting-pchronicle +git commit -m "refactor: remove pchronicle production unwraps" +``` + +--- + +### Task 5: Enable Strict Clippy Locally and in CI + +**Files:** +- Modify: `crates/persisting-pvisor/src/cli/run.rs` +- Modify: `justfile` +- Modify: `.github/workflows/ci.yml` + +**Interfaces:** +- Produces: strict active-workspace Clippy and a repeatable pChronicle production panic check. +- Preserves: the independent strict `persisting-dlcapt` workflow. + +- [ ] **Step 1: Run strict active-workspace Clippy and verify RED** + +Run: + +```bash +cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings +``` + +Expected: FAIL on the Chronicle sink tuple in `persisting-pvisor/src/cli/run.rs` with `clippy::type_complexity`. + +- [ ] **Step 2: Name the pVisor sink tuple** + +Add this module-level alias near the Chronicle imports: + +```rust +type ChronicleSinks = ( + Arc, + Arc, + Option, + Option>, +); +``` + +Use `let (sink, event_sink, writer, chronicle_control): ChronicleSinks = ...` without a lint suppression. + +- [ ] **Step 3: Make local lint commands strict** + +Define the recipes so `lint-rust` runs both guards: + +```make +lint-rust: clippy-deny clippy-pchronicle-panics + +clippy-deny: + cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings + +clippy-pchronicle-panics: + cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used + +clippy: + just lint-rust +``` + +Remove comments that describe strict Clippy as unsafe before cleanup. + +- [ ] **Step 4: Mirror both guards in CI** + +Replace the ordinary Rust Clippy step with: + +```yaml +- name: Rust clippy + run: | + cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings + cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used +``` + +- [ ] **Step 5: Verify strict lint locally** + +Run: + +```bash +cargo fmt --all -- --check +just lint-rust +``` + +Expected: both commands pass with no warnings. + +- [ ] **Step 6: Commit strict lint enforcement** + +```bash +git add crates/persisting-pvisor/src/cli/run.rs justfile .github/workflows/ci.yml +git commit -m "ci: deny active workspace clippy warnings" +``` + +--- + +### Task 6: Final Cross-Boundary Verification + +**Files:** +- Verify only; modify a prior task's owning files if a failure exposes an omission. + +**Interfaces:** +- Confirms: the complete removal, panic policy, tests, and strict lint policy work together. + +- [ ] **Step 1: Verify removal and dependency boundaries** + +Run: + +```bash +! git grep -n -i -E 'judge|judgment' -- crates/persisting-pchronicle crates/persisting-pchronicle-cli pchronicle-web/src pchronicle-web/assets docs/src +! rg -n '^reqwest\s*=' crates/persisting-pchronicle/Cargo.toml +cargo tree -p persisting-pchronicle --depth 1 --locked +``` + +Expected: both scans pass; the direct dependency tree does not list `reqwest` as a pChronicle dependency. Transitive HTTP dependencies brought by Lance are allowed. + +- [ ] **Step 2: Run all targeted tests** + +Run: + +```bash +cargo test -p persisting-pchronicle --lib --locked +cargo test -p persisting-pchronicle-cli --lib --tests --locked +cargo test --manifest-path pchronicle-web/Cargo.toml --locked +``` + +Expected: all tests pass with zero failures. + +- [ ] **Step 3: Run all formatting and lint gates** + +Run: + +```bash +cargo fmt --all -- --check +cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check +just lint-rust +git diff --check +``` + +Expected: all commands exit zero and Clippy prints no warnings. + +- [ ] **Step 4: Review the final diff and status** + +Run: + +```bash +git status --short +git diff --stat HEAD~5..HEAD +git log -5 --oneline +``` + +Confirm that unrelated untracked user files remain untouched and every tracked change belongs to this design. From aa442213332a81edfe0f2e1239388fcac6c22507 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 19:50:13 +0800 Subject: [PATCH 04/65] refactor: remove pchronicle judge subsystem --- .github/workflows/ci.yml | 6 +- Cargo.lock | 1 - .../src/server/explorer.rs | 113 +--- .../src/server/mod.rs | 73 +-- .../src/server/tests.rs | 121 +--- .../tests/server_http_contract.rs | 1 - crates/persisting-pchronicle/Cargo.toml | 1 - .../persisting-pchronicle/src/convert/actf.rs | 12 +- .../persisting-pchronicle/src/formats/actf.rs | 13 +- .../src/formats/openai_corpus.rs | 9 +- .../src/judge_service.rs | 307 --------- crates/persisting-pchronicle/src/judgment.rs | 602 ------------------ .../src/judgment_summary.rs | 269 -------- .../src/layout/coords.rs | 29 - .../persisting-pchronicle/src/layout/mod.rs | 2 +- crates/persisting-pchronicle/src/lib.rs | 29 +- crates/persisting-pchronicle/src/messages.rs | 196 ------ .../src/operations/trajectory/judge.rs | 62 -- .../src/operations/trajectory/judge_stats.rs | 70 -- .../src/operations/trajectory/mod.rs | 60 +- crates/persisting-pchronicle/src/revision.rs | 20 +- .../src/store/catalog/discovery.rs | 7 +- .../src/store/catalog/provider.rs | 6 +- .../src/store/catalog/tests.rs | 4 +- .../src/store/index_build_gate.rs | 9 +- .../src/store/root_write_lock.rs | 4 +- crates/persisting-pvisor/src/cli/run.rs | 15 +- .../concepts/facts-and-projections.md | 4 +- .../concepts/facts-and-projections.zh.md | 4 +- docs/src/pchronicle/design/catalog.md | 8 +- docs/src/pchronicle/design/catalog.zh.md | 8 +- .../pchronicle/design/trajectory-storage.md | 9 +- .../design/trajectory-storage.zh.md | 9 +- docs/src/rfcs/0002-events-format.md | 2 +- docs/src/rfcs/0003-pchronicle-ownership.md | 3 +- .../rfcs/0005-pchronicle-revision-lineage.md | 4 +- justfile | 9 +- pchronicle-web/assets/workbench.css | 2 +- pchronicle-web/src/agent.rs | 49 +- pchronicle-web/src/api.rs | 15 +- pchronicle-web/src/components.rs | 10 - pchronicle-web/src/model.rs | 17 - pchronicle-web/src/workspace.rs | 173 +---- 43 files changed, 147 insertions(+), 2220 deletions(-) delete mode 100644 crates/persisting-pchronicle/src/judge_service.rs delete mode 100644 crates/persisting-pchronicle/src/judgment.rs delete mode 100644 crates/persisting-pchronicle/src/judgment_summary.rs delete mode 100644 crates/persisting-pchronicle/src/operations/trajectory/judge.rs delete mode 100644 crates/persisting-pchronicle/src/operations/trajectory/judge_stats.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6643a2f5..fa2c1747 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,9 @@ jobs: cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check - name: Rust clippy - # Match local `just clippy` for now; tighten to `-D warnings` once the - # workspace is clean (same bar as Pulsing). - run: cargo clippy --workspace --all-targets --locked + run: | + cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings + cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used - name: Python lint (ruff) run: uvx ruff check persisting/ diff --git a/Cargo.lock b/Cargo.lock index 01dd5244..34618ddb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6972,7 +6972,6 @@ dependencies = [ "object_store", "persisting-agentctl", "persisting-events", - "reqwest 0.12.28", "serde", "serde_json", "serde_yaml", diff --git a/crates/persisting-pchronicle-cli/src/server/explorer.rs b/crates/persisting-pchronicle-cli/src/server/explorer.rs index 2e189611..d3f6ab3e 100644 --- a/crates/persisting-pchronicle-cli/src/server/explorer.rs +++ b/crates/persisting-pchronicle-cli/src/server/explorer.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use persisting_pchronicle::{EventRecord, JudgeRow}; +use persisting_pchronicle::EventRecord; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -13,7 +13,6 @@ pub(crate) struct ExplorerRunsQuery { pub(crate) status: Option, pub(crate) agent: Option, pub(crate) model: Option, - pub(crate) verdict: Option, pub(crate) path: Option, pub(crate) sort: Option, pub(crate) direction: Option, @@ -48,9 +47,6 @@ pub(crate) struct RunExplorerItem { #[serde(flatten)] pub(crate) run: RunSummary, pub(crate) model: Option, - pub(crate) judgment_count: usize, - pub(crate) average_score: Option, - pub(crate) verdict: Option, } #[derive(Clone, Debug, Serialize)] @@ -111,8 +107,6 @@ pub(crate) struct RunAnalysis { pub(crate) kind_breakdown: Vec, pub(crate) model_breakdown: Vec, pub(crate) tools: Vec, - pub(crate) judgment_count: usize, - pub(crate) average_score: Option, } #[derive(Clone, Debug, Serialize)] @@ -132,30 +126,6 @@ pub(crate) struct TurnSummary { pub(crate) tool_names: Vec, pub(crate) event_seqs: Vec, pub(crate) has_error: bool, - pub(crate) judgment_count: usize, -} - -#[derive(Clone, Debug, Serialize)] -pub(crate) struct JudgmentView { - pub(crate) session_id: String, - pub(crate) call_id: String, - pub(crate) rubric_id: String, - pub(crate) score: i64, - pub(crate) verdict: String, - pub(crate) rationale: String, -} - -impl From<&JudgeRow> for JudgmentView { - fn from(row: &JudgeRow) -> Self { - Self { - session_id: row.session_id.clone(), - call_id: row.call_id.clone(), - rubric_id: row.rubric_id.clone(), - score: row.score, - verdict: row.verdict.clone(), - rationale: row.rationale.clone(), - } - } } #[derive(Clone, Debug, Serialize)] @@ -164,14 +134,9 @@ pub(crate) struct TurnDetail { pub(crate) turn: persisting_pchronicle::StorylineTurn, pub(crate) wire_tool_calls: Vec, pub(crate) events: Vec, - pub(crate) judgments: Vec, } -pub(crate) fn run_page( - summaries: Vec, - judgments: &BTreeMap>, - query: &ExplorerRunsQuery, -) -> RunExplorerPage { +pub(crate) fn run_page(summaries: Vec, query: &ExplorerRunsQuery) -> RunExplorerPage { let needle = query .q .as_deref() @@ -180,18 +145,9 @@ pub(crate) fn run_page( .to_ascii_lowercase(); let mut records = summaries .into_iter() - .map(|run| { - let rows = judgments.get(&run_key(&run)).cloned().unwrap_or_default(); - let average_score = average_score(&rows); - let verdict = aggregate_verdict(&rows); - let model = run.model_name.clone(); - RunExplorerItem { - run, - model, - judgment_count: rows.len(), - average_score, - verdict, - } + .map(|run| RunExplorerItem { + model: run.model_name.clone(), + run, }) .filter(|item| { (needle.is_empty() @@ -212,10 +168,6 @@ pub(crate) fn run_page( item.model.as_deref().unwrap_or_default(), query.model.as_deref(), ) - && matches_filter( - item.verdict.as_deref().unwrap_or_default(), - query.verdict.as_deref(), - ) }) .collect::>(); let path_index = records.iter().map(|item| item.run.clone()).collect(); @@ -232,10 +184,6 @@ pub(crate) fn run_page( records.sort_by( |left, right| match query.sort.as_deref().unwrap_or("session") { "events" => left.run.row_count.cmp(&right.run.row_count), - "score" => left - .average_score - .partial_cmp(&right.average_score) - .unwrap_or(std::cmp::Ordering::Equal), "status" => left.run.status.cmp(&right.run.status), "agent" => left.run.agent_id.cmp(&right.run.agent_id), _ => left.run.session_id.cmp(&right.run.session_id), @@ -261,7 +209,6 @@ pub(crate) fn analyze( run: RunSummary, turns: &[TrajectoryTurnView], events: &[EventRecord], - judgments: &[JudgeRow], ) -> RunAnalysis { let mut latencies = Vec::new(); let mut ttfts = Vec::new(); @@ -436,8 +383,6 @@ pub(crate) fn analyze( kind_breakdown: dimension_aggregates(kinds), model_breakdown: dimension_aggregates(model_groups), tools, - judgment_count: judgments.len(), - average_score: average_score(judgments), run, } } @@ -445,7 +390,6 @@ pub(crate) fn analyze( pub(crate) fn turn_page( turns: &[TrajectoryTurnView], events: &[EventRecord], - judgments: &[JudgeRow], q: Option<&str>, source: Option<&str>, offset: usize, @@ -456,44 +400,26 @@ pub(crate) fn turn_page( .iter() .filter(|item| source.is_none_or(|source| source == "all" || item.turn.source == source)) .filter(|item| needle.is_empty() || searchable_turn(item).contains(&needle)) - .map(|item| turn_summary(item, events, judgments)) + .map(|item| turn_summary(item, events)) .collect(); paginate(records, offset, limit.clamp(1, 500)) } -pub(crate) fn turn_detail( - item: &TrajectoryTurnView, - events: &[EventRecord], - judgments: &[JudgeRow], -) -> TurnDetail { +pub(crate) fn turn_detail(item: &TrajectoryTurnView, events: &[EventRecord]) -> TurnDetail { let linked = events .iter() .filter(|event| item.event_seqs.contains(&event.seq)) .cloned() .collect::>(); - let scoped = judgments - .iter() - .filter(|row| item.call_id.as_deref() == Some(row.call_id.as_str())) - .map(JudgmentView::from) - .collect(); TurnDetail { - summary: turn_summary(item, events, judgments), + summary: turn_summary(item, events), turn: item.turn.clone(), wire_tool_calls: item.wire_tool_calls.clone(), events: linked, - judgments: scoped, } } -pub(crate) fn run_key(run: &RunSummary) -> String { - format!("{}\u{1f}{}\u{1f}{}", run.dataset, run.file, run.session_id) -} - -fn turn_summary( - item: &TrajectoryTurnView, - events: &[EventRecord], - judgments: &[JudgeRow], -) -> TurnSummary { +fn turn_summary(item: &TrajectoryTurnView, events: &[EventRecord]) -> TurnSummary { let linked = events .iter() .filter(|event| item.event_seqs.contains(&event.seq)) @@ -549,10 +475,6 @@ fn turn_summary( .collect(), event_seqs: item.event_seqs.clone(), has_error: turn_has_error(item, &linked), - judgment_count: judgments - .iter() - .filter(|row| item.call_id.as_deref() == Some(row.call_id.as_str())) - .count(), } } @@ -584,23 +506,6 @@ fn matches_filter(value: &str, filter: Option<&str>) -> bool { .is_none_or(|filter| value.eq_ignore_ascii_case(filter)) } -fn aggregate_verdict(rows: &[JudgeRow]) -> Option { - if rows.is_empty() { - None - } else if rows.iter().any(|row| row.verdict == "fail") { - Some("fail".into()) - } else if rows.iter().any(|row| row.verdict == "partial") { - Some("partial".into()) - } else { - Some("pass".into()) - } -} - -fn average_score(rows: &[JudgeRow]) -> Option { - (!rows.is_empty()) - .then(|| rows.iter().map(|row| row.score as f64).sum::() / rows.len() as f64) -} - #[derive(Default)] struct ToolAccumulator { count: usize, diff --git a/crates/persisting-pchronicle-cli/src/server/mod.rs b/crates/persisting-pchronicle-cli/src/server/mod.rs index 10df12b0..04ace98f 100644 --- a/crates/persisting-pchronicle-cli/src/server/mod.rs +++ b/crates/persisting-pchronicle-cli/src/server/mod.rs @@ -15,9 +15,9 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use persisting_pchronicle::{ - events_to_har, events_to_otlp_json, read_judge_rows, read_revisions, CatalogErrorPolicy, - CatalogSnapshotOptions, CatalogStorylineKey, ChronicleQueryEngine, DatasetCatalogSnapshot, - DatasetMount, EventRecord, JudgeRow, StoryCoords, StorylineTurn, DEFAULT_DATASET_NAME, + events_to_har, events_to_otlp_json, read_revisions, CatalogErrorPolicy, CatalogSnapshotOptions, + CatalogStorylineKey, ChronicleQueryEngine, DatasetCatalogSnapshot, DatasetMount, EventRecord, + StoryCoords, StorylineTurn, DEFAULT_DATASET_NAME, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -141,7 +141,6 @@ fn read_routes() -> Router { .route("/api/trajectory-view", get(trajectory_view)) .route("/api/export/har", get(export_har)) .route("/api/export/otlp", get(export_otlp)) - .route("/api/judgments", get(judgments)) .route("/api/revisions", get(revisions)) .route("/api/catalog", get(catalog).post(refresh_catalog)) .route("/api/query/tables", get(query_tables)) @@ -300,28 +299,7 @@ async fn explorer_runs( Query(query): Query, ) -> Result, ApiError> { let summaries = load_run_summaries(&state).await?; - let mut judgments_by_run = BTreeMap::new(); - for run in &summaries { - let session_query = SessionQuery { - dataset: Some(run.dataset.clone()), - file: Some(run.file.clone()), - run_id: Some(run.run_id.clone()), - agent_id: run.agent_id.clone(), - session_id: run.session_id.clone(), - root_session_id: run.root_session_id.clone(), - offset: None, - limit: None, - }; - let rows = session_judgments(&state, &session_query) - .await - .unwrap_or_default(); - judgments_by_run.insert(explorer::run_key(run), rows); - } - Ok(Json(explorer::run_page( - summaries, - &judgments_by_run, - &query, - ))) + Ok(Json(explorer::run_page(summaries, &query))) } async fn resolve_run_summary( @@ -710,32 +688,15 @@ async fn trajectory_view( })) } -async fn session_judgments( - state: &AppState, - query: &SessionQuery, -) -> Result, ApiError> { - let Some(coords) = canonical_run_coords(state, query).await? else { - return Ok(Vec::new()); - }; - Ok(read_judge_rows(&coords) - .await - .map_err(api_error)? - .into_iter() - .filter(|row| row.session_id == query.session_id) - .collect()) -} - async fn explorer_run( State(state): State, Query(query): Query, ) -> Result, ApiError> { let loaded = load_trajectory(&state, &query).await?; - let judgments = session_judgments(&state, &query).await?; Ok(Json(explorer::analyze( loaded.run, &loaded.turns, &loaded.records, - &judgments, ))) } @@ -774,11 +735,9 @@ async fn explorer_turns( ) -> Result>, ApiError> { let session = query.session(); let loaded = load_trajectory(&state, &session).await?; - let judgments = session_judgments(&state, &session).await?; Ok(Json(explorer::turn_page( &loaded.turns, &loaded.records, - &judgments, query.q.as_deref(), query.source.as_deref(), query.offset.unwrap_or(0), @@ -820,12 +779,7 @@ async fn explorer_turn( code: "turn_not_found", message: format!("turn {} was not found", query.turn_id), })?; - let judgments = session_judgments(&state, &session).await?; - Ok(Json(explorer::turn_detail( - item, - &loaded.records, - &judgments, - ))) + Ok(Json(explorer::turn_detail(item, &loaded.records))) } async fn export_har( @@ -844,23 +798,6 @@ async fn export_otlp( ))) } -async fn judgments( - State(state): State, - Query(query): Query, -) -> Result, ApiError> { - let rows = session_judgments(&state, &query).await?; - Ok(Json(Value::Array( - rows.into_iter() - .map(|row| { - json!({ - "session_id":row.session_id,"call_id":row.call_id,"rubric_id":row.rubric_id, - "score":row.score,"verdict":row.verdict,"rationale":row.rationale - }) - }) - .collect(), - ))) -} - async fn revisions( State(state): State, Query(query): Query, diff --git a/crates/persisting-pchronicle-cli/src/server/tests.rs b/crates/persisting-pchronicle-cli/src/server/tests.rs index 028135ba..c1ae51f1 100644 --- a/crates/persisting-pchronicle-cli/src/server/tests.rs +++ b/crates/persisting-pchronicle-cli/src/server/tests.rs @@ -1,6 +1,5 @@ use super::*; use axum::http::header; -use persisting_pchronicle::write_judge_rows; fn router(storage: impl Into) -> Router { let config = ChronicleServerConfig::mounted(vec![ @@ -539,82 +538,7 @@ async fn explorer_uses_terminal_metadata_for_run_status() { } #[tokio::test] -async fn read_only_mounts_expose_existing_judgments() -> anyhow::Result<()> { - use http_body_util::BodyExt; - use persisting_pchronicle::RawEventLanceStore; - use tower::ServiceExt; - - let root = tempfile::tempdir()?; - let coords = StoryCoords::new( - root.path().to_string_lossy(), - "agent", - "child-session", - Some("shared-run".into()), - ); - RawEventLanceStore - .append_events( - &coords, - &[EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("child-session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"content":"captured"}), - }], - ) - .await?; - write_judge_rows( - &coords, - &[JudgeRow { - session_id: "child-session".into(), - call_id: "__story__".into(), - rubric_id: "quality".into(), - score: 91, - verdict: "pass".into(), - rationale: "stored before mounting read-only".into(), - }], - ) - .await?; - - let app = test_router_with_config(ChronicleServerConfig::mounted(vec![DatasetMount::new( - "archive", - root.path().to_string_lossy(), - )?])?); - let response = app - .oneshot( - axum::http::Request::builder() - .uri( - "/api/judgments?dataset=archive&file=agent%2Fshared-run%2Fevents.lance&run_id=shared-run&agent_id=agent&session_id=child-session", - ) - .body(axum::body::Body::empty())?, - ) - .await?; - let response_status = response.status(); - let response_body = response.into_body().collect().await?.to_bytes(); - assert_eq!( - response_status, - StatusCode::OK, - "judgment read failed: {}", - String::from_utf8_lossy(&response_body) - ); - let rows: Value = serde_json::from_slice(&response_body)?; - assert_eq!(rows[0]["score"], 91); - assert_eq!(rows[0]["session_id"], "child-session"); - Ok(()) -} - -#[tokio::test] -async fn limited_query_and_read_only_judgments_enforce_copilot_boundaries() { +async fn limited_query_enforces_copilot_boundaries() { use http_body_util::BodyExt; use tower::ServiceExt; @@ -655,49 +579,6 @@ async fn limited_query_and_read_only_judgments_enforce_copilot_boundaries() { assert_eq!(evidence["max_rows"], 1); assert_eq!(evidence["max_bytes"], 1_048_576); - let write = app - .clone() - .oneshot( - axum::http::Request::builder() - .method("POST") - .uri("/api/judgments") - .header(header::CONTENT_TYPE, "application/json") - .body(axum::body::Body::from( - json!({ - "agent_id":"model-json","session_id":"json-session", - "root_session_id":"json-job","call_id":"__story__", - "rubric_id":"quality","score":88,"verdict":"pass", - "rationale":"Evidence supports the trajectory-level verdict." - }) - .to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(write.status(), StatusCode::METHOD_NOT_ALLOWED); - - let saved = app - .clone() - .oneshot( - axum::http::Request::builder() - .uri("/api/judgments?agent_id=model-json&session_id=json-session&root_session_id=json-job") - .body(axum::body::Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let saved_status = saved.status(); - let saved_body = saved.into_body().collect().await.unwrap().to_bytes(); - assert_eq!( - saved_status, - StatusCode::OK, - "judgments failed: {}", - String::from_utf8_lossy(&saved_body) - ); - let saved: Value = serde_json::from_slice(&saved_body).unwrap(); - assert_eq!(saved, json!([])); - std::fs::remove_dir_all(root).unwrap(); } diff --git a/crates/persisting-pchronicle-cli/tests/server_http_contract.rs b/crates/persisting-pchronicle-cli/tests/server_http_contract.rs index 3c2291f2..a9be4e86 100644 --- a/crates/persisting-pchronicle-cli/tests/server_http_contract.rs +++ b/crates/persisting-pchronicle-cli/tests/server_http_contract.rs @@ -69,7 +69,6 @@ async fn warehouse_read_route_matrix_exposes_the_documented_surface() -> Result< async fn warehouse_write_route_matrix_never_exposes_dataset_mutations() -> Result<()> { let app = warehouse()?; for (method, path) in [ - (Method::POST, "/api/judgments"), (Method::POST, "/api/maintain"), (Method::POST, "/api/query"), (Method::PUT, "/api/events"), diff --git a/crates/persisting-pchronicle/Cargo.toml b/crates/persisting-pchronicle/Cargo.toml index 549a6000..00df5cc5 100644 --- a/crates/persisting-pchronicle/Cargo.toml +++ b/crates/persisting-pchronicle/Cargo.toml @@ -50,7 +50,6 @@ lance-table = { workspace = true, optional = true } object_store = { workspace = true, optional = true } persisting-agentctl.workspace = true persisting-events.workspace = true -reqwest = { workspace = true, features = ["json", "rustls-tls"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } serde_yaml.workspace = true diff --git a/crates/persisting-pchronicle/src/convert/actf.rs b/crates/persisting-pchronicle/src/convert/actf.rs index dc898f27..97be8601 100644 --- a/crates/persisting-pchronicle/src/convert/actf.rs +++ b/crates/persisting-pchronicle/src/convert/actf.rs @@ -74,7 +74,8 @@ pub fn storylines_to_actf(stories: &[StorylineDocument]) -> Result )); } - let first = provenance(&stories[0]).expect("provenance count checked above"); + let first = provenance(&stories[0]) + .ok_or_else(|| Error::Other("ACTF provenance disappeared during conversion".into()))?; let root_value = first .get("root") .and_then(Value::as_object) @@ -83,7 +84,8 @@ pub fn storylines_to_actf(stories: &[StorylineDocument]) -> Result let mut attempts = Map::new(); for story in stories { story.validate()?; - let metadata = provenance(story).expect("provenance count checked above"); + let metadata = provenance(story) + .ok_or_else(|| Error::Other("ACTF provenance disappeared during conversion".into()))?; if metadata.get("root").and_then(Value::as_object) != Some(&root_value) { return Err(Error::Other( "ACTF Storylines have conflicting root metadata".into(), @@ -450,7 +452,7 @@ fn root_metadata(document: &ActfDocument) -> Result { let mut value = serde_json::to_value(document)?; value .as_object_mut() - .expect("ActfDocument serializes as an object") + .ok_or_else(|| Error::Other("serialized ACTF document must be an object".into()))? .remove("attempts"); Ok(value) } @@ -459,7 +461,7 @@ fn attempt_metadata(attempt: &ActfAttempt) -> Result { let mut value = serde_json::to_value(attempt)?; value .as_object_mut() - .expect("ActfAttempt serializes as an object") + .ok_or_else(|| Error::Other("serialized ACTF attempt must be an object".into()))? .remove("trajectory"); Ok(value) } @@ -468,7 +470,7 @@ fn trajectory_metadata(trajectory: &ActfTrajectory) -> Result { let mut value = serde_json::to_value(trajectory)?; value .as_object_mut() - .expect("ActfTrajectory serializes as an object") + .ok_or_else(|| Error::Other("serialized ACTF trajectory must be an object".into()))? .remove("steps"); Ok(value) } diff --git a/crates/persisting-pchronicle/src/formats/actf.rs b/crates/persisting-pchronicle/src/formats/actf.rs index d0dffafb..d7791c90 100644 --- a/crates/persisting-pchronicle/src/formats/actf.rs +++ b/crates/persisting-pchronicle/src/formats/actf.rs @@ -254,12 +254,13 @@ impl ActfTrajectory { .get("tool_use_id") .or_else(|| observation.extra.get("id")) .and_then(Value::as_str); - if referenced_id.is_some_and(|id| !step_call_ids.contains(id)) { - return Err(Error::Other(format!( - "ACTF step {} observation references unknown tool id '{}'", - step.step_id, - referenced_id.expect("checked above") - ))); + if let Some(referenced_id) = referenced_id { + if !step_call_ids.contains(referenced_id) { + return Err(Error::Other(format!( + "ACTF step {} observation references unknown tool id '{}'", + step.step_id, referenced_id + ))); + } } } } diff --git a/crates/persisting-pchronicle/src/formats/openai_corpus.rs b/crates/persisting-pchronicle/src/formats/openai_corpus.rs index 80951e6f..03a27016 100644 --- a/crates/persisting-pchronicle/src/formats/openai_corpus.rs +++ b/crates/persisting-pchronicle/src/formats/openai_corpus.rs @@ -310,9 +310,12 @@ fn rows_to_storyline( let mut next_turn_id = 1_i64; for (ordinal, raw) in records { - let row = raw - .as_object() - .expect("rows were validated as objects during grouping"); + let row = raw.as_object().ok_or_else(|| { + Error::Other(format!( + "OpenAI corpus {} row {} must be an object", + relative_path, ordinal + )) + })?; let step_id = row.get("step_id").and_then(Value::as_i64).ok_or_else(|| { Error::Other(format!( "OpenAI corpus {} row {} requires integer step_id", diff --git a/crates/persisting-pchronicle/src/judge_service.rs b/crates/persisting-pchronicle/src/judge_service.rs deleted file mode 100644 index fe8b3192..00000000 --- a/crates/persisting-pchronicle/src/judge_service.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! Protocol-independent trajectory judgment orchestration. - -use anyhow::{Context, Result}; - -use crate::{ - build_llm_judge_prompt, dry_run_judge_rows, evaluation_units, judgment_dataset_path, - manual_few_shot_examples, manual_judge_rows, parse_llm_judge_rows, pending_evaluation_units, - read_judge_rows, write_judge_rows, JudgeRow, JudgmentScope, ManualJudgmentInput, - RawEventLanceStore, StoryCoords, -}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum JudgingMethod { - Manual, - Llm, -} - -#[derive(Debug, Clone)] -pub struct JudgeTrajectoryRequest { - pub session: StoryCoords, - pub rubric_id: String, - pub rubric_ids: Vec, - pub scope: JudgmentScope, - pub method: JudgingMethod, - pub force: bool, - pub dry_run: bool, - pub model: Option, - pub few_shot_limit: usize, - pub manual_scores: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JudgeTrajectoryOutcome { - pub primary_rubric: String, - pub rubric_ids: Vec, - pub dataset: String, - pub judged_units: usize, - pub skipped_units: usize, - pub status: String, - pub note: String, -} - -pub async fn judge_trajectory(request: JudgeTrajectoryRequest) -> Result { - let rubric_ids = resolve_rubric_ids(&request.rubric_id, &request.rubric_ids); - let primary_rubric = rubric_ids - .first() - .cloned() - .unwrap_or_else(|| "default".into()); - - let store = RawEventLanceStore; - if !store.exists(&request.session).await? { - anyhow::bail!( - "Lance event log missing for session {}; judge requires events.lance", - request.session.session_id - ); - } - let records = store.read_events(&request.session, 0, None).await?; - let units = evaluation_units(&records, request.scope)?; - if units.is_empty() { - anyhow::bail!( - "no judge units for session {} ({:?})", - request.session.session_id, - request.scope - ); - } - - let existing = read_judge_rows(&request.session).await?; - let mut judged_units = 0; - let mut skipped_units = 0; - let mut incoming = Vec::new(); - - for rubric_id in &rubric_ids { - let (pending, skipped) = pending_evaluation_units( - &existing, - &request.session.session_id, - rubric_id, - &units, - request.force, - ); - skipped_units += skipped; - if pending.is_empty() { - continue; - } - - let rows = match request.method { - JudgingMethod::Manual => manual_judge_rows( - &request.session.session_id, - rubric_id, - &pending, - &request.manual_scores, - )?, - JudgingMethod::Llm if request.dry_run => { - dry_run_judge_rows(&request.session.session_id, rubric_id, &pending) - } - JudgingMethod::Llm => { - let examples = - manual_few_shot_examples(&existing, rubric_id, request.few_shot_limit); - llm_judge_rows( - &request.session, - request.scope, - rubric_id, - request.model.as_deref(), - &pending, - &examples, - ) - .await? - } - }; - judged_units += pending.len(); - incoming.extend(rows); - } - - let dataset = if incoming.is_empty() { - judgment_dataset_path(&request.session).await? - } else { - write_judge_rows(&request.session, &incoming).await? - }; - Ok(JudgeTrajectoryOutcome { - primary_rubric, - rubric_ids: rubric_ids.clone(), - dataset: dataset.clone(), - judged_units, - skipped_units, - status: "ok".into(), - note: format!( - "Judge {:?}/{:?}: {} rubric(s), {} unit(s) scored, {} skipped. Judgments stored in {}.", - request.method, - request.scope, - rubric_ids.len(), - judged_units, - skipped_units, - dataset - ), - }) -} - -fn resolve_rubric_ids(single: &str, multiple: &[String]) -> Vec { - if !multiple.is_empty() { - return multiple - .iter() - .map(|rubric| { - if rubric.trim().is_empty() { - "default".into() - } else { - rubric.trim().into() - } - }) - .collect(); - } - vec![if single.trim().is_empty() { - "default".into() - } else { - single.into() - }] -} - -async fn llm_judge_rows( - session: &StoryCoords, - scope: JudgmentScope, - rubric_id: &str, - model: Option<&str>, - units: &[crate::EvaluationUnit], - few_shot: &[JudgeRow], -) -> Result> { - let model = model - .map(str::to_string) - .or_else(|| std::env::var("PERSISTING_JUDGE_MODEL").ok()) - .unwrap_or_else(|| "gpt-4o-mini".into()); - let prompt = build_llm_judge_prompt(scope, rubric_id, units, few_shot); - let output = call_openai_chat(&model, &prompt).await?; - parse_llm_judge_rows(&session.session_id, rubric_id, &output) -} - -async fn call_openai_chat(model: &str, user_prompt: &str) -> Result { - let base = std::env::var("OPENAI_BASE_URL") - .or_else(|_| std::env::var("PERSISTING_JUDGE_BASE_URL")) - .unwrap_or_else(|_| "https://api.openai.com/v1".into()); - let api_key = std::env::var("OPENAI_API_KEY") - .or_else(|_| std::env::var("PERSISTING_JUDGE_API_KEY")) - .context("OPENAI_API_KEY (or PERSISTING_JUDGE_API_KEY) required for judge")?; - let response = reqwest::Client::builder() - .build() - .context("build reqwest client for judge")? - .post(format!("{}/chat/completions", base.trim_end_matches('/'))) - .bearer_auth(api_key) - .json(&serde_json::json!({ - "model": model, - "temperature": 0, - "response_format": { "type": "json_object" }, - "messages": [ - {"role": "system", "content": "You output strict JSON only."}, - {"role": "user", "content": user_prompt} - ] - })) - .send() - .await - .context("judge LLM HTTP request")?; - let status = response.status(); - let text = response.text().await.context("read judge LLM response")?; - if !status.is_success() { - anyhow::bail!("judge LLM HTTP {status}: {text}"); - } - let value: serde_json::Value = - serde_json::from_str(&text).context("parse judge LLM envelope")?; - value["choices"][0]["message"]["content"] - .as_str() - .map(str::to_string) - .ok_or_else(|| anyhow::anyhow!("judge LLM response missing message content")) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{EventRecord, RawEventLanceStore}; - - fn event(kind: &str, content_key: &str, content: &str) -> EventRecord { - EventRecord { - identity: crate::EventIdentity::default(), - seq: 0, - source: "test".into(), - kind: kind.into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: Some("call-1".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({(content_key): content}), - } - } - - #[tokio::test] - async fn manual_judge_isolated_from_append_only_events() { - let dir = tempfile::tempdir().unwrap(); - let session = StoryCoords::new(dir.path().to_string_lossy(), "agent", "session", None); - RawEventLanceStore - .append_events( - &session, - &[ - event("llm.request", "user_content", "hello"), - event("llm.response", "assistant_content", "world"), - ], - ) - .await - .unwrap(); - - let outcome = judge_trajectory(JudgeTrajectoryRequest { - session: session.clone(), - rubric_id: "quality".into(), - rubric_ids: Vec::new(), - scope: JudgmentScope::Story, - method: JudgingMethod::Manual, - force: false, - dry_run: false, - model: None, - few_shot_limit: 0, - manual_scores: vec![ManualJudgmentInput { - call_id: Some(crate::STORY_CALL_ID.into()), - rubric_id: "quality".into(), - score: 90, - verdict: "pass".into(), - rationale: "good".into(), - }], - }) - .await - .unwrap(); - - assert_eq!(outcome.judged_units, 1); - let rows = crate::read_judge_rows(&session).await.unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].score, 90); - assert!(rows[0] - .rationale - .starts_with(crate::MANUAL_RATIONALE_PREFIX)); - - let event_layout = RawEventLanceStore.layout_stats(&session).await.unwrap(); - assert_eq!(event_layout.visible_rows, 2); - assert!(crate::raw_event_arrow_schema() - .fields - .iter() - .all(|field| !field.name().starts_with("judge_"))); - assert!(crate::judgment_dataset_path(&session) - .await - .unwrap() - .ends_with("judgments.lance")); - - RawEventLanceStore - .append_events(&session, &[event("note", "content", "after judgment")]) - .await - .unwrap(); - RawEventLanceStore - .append_events( - &session, - &[ - event("llm.request", "user_content", "hello again"), - event("llm.response", "assistant_content", "world again"), - ], - ) - .await - .unwrap(); - assert_eq!(crate::read_judge_rows(&session).await.unwrap().len(), 1); - } -} diff --git a/crates/persisting-pchronicle/src/judgment.rs b/crates/persisting-pchronicle/src/judgment.rs deleted file mode 100644 index e78f335c..00000000 --- a/crates/persisting-pchronicle/src/judgment.rs +++ /dev/null @@ -1,602 +0,0 @@ -//! Normalized judge results stored at `{run}/judgments.lance`. -//! -//! The canonical event schema never evolves when a rubric is added. - -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use futures::TryStreamExt; -use lance::dataset::{InsertBuilder, MergeInsertBuilder, WhenMatched, WhenNotMatched}; -use lance::deps::arrow_array::{Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; -use lance::deps::arrow_schema::{DataType, Field, Schema as ArrowSchema}; -use lance::Dataset; - -use crate::{story_lance_judgment_path, StoryCoords}; - -pub const STORY_CALL_ID: &str = "__story__"; -pub const MANUAL_RATIONALE_PREFIX: &str = "[manual] "; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JudgeRow { - pub session_id: String, - pub call_id: String, - pub rubric_id: String, - pub score: i64, - pub verdict: String, - pub rationale: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JudgeDialogueUnit { - pub call_id: String, - pub user: String, - pub assistant: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum JudgmentScope { - Story, - Turn, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EvaluationUnit { - pub call_id: String, - pub body: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManualJudgmentInput { - pub call_id: Option, - pub rubric_id: String, - pub score: i64, - pub verdict: String, - pub rationale: String, -} - -/// Build complete user/assistant units from canonical events using pChronicle's -/// dialogue projection policy. -pub fn dialogue_judge_units(records: &[crate::EventRecord]) -> Result> { - let (blocks, _) = crate::event_records_to_markdown_blocks(records)?; - let mut units: Vec<(String, Option, Option)> = Vec::new(); - let mut by_call_id = HashMap::new(); - - for block in blocks { - let Some(role) = block.role() else { - continue; - }; - let Some(call_id) = block - .header - .fields - .get("call_id") - .and_then(|value| value.as_str()) - .map(str::to_string) - else { - continue; - }; - let index = *by_call_id.entry(call_id.clone()).or_insert_with(|| { - units.push((call_id, None, None)); - units.len() - 1 - }); - match role { - "user" => units[index].1 = Some(block.body), - "assistant" => units[index].2 = Some(block.body), - _ => {} - } - } - - Ok(units - .into_iter() - .filter_map(|(call_id, user, assistant)| { - Some(JudgeDialogueUnit { - call_id, - user: user?, - assistant: assistant?, - }) - }) - .collect()) -} - -pub fn story_judge_body(units: &[JudgeDialogueUnit]) -> String { - let mut body = String::new(); - for (index, unit) in units.iter().enumerate() { - body.push_str(&format!( - "### Turn {} (call_id={})\nUser:\n{}\n\nAssistant:\n{}\n\n", - index + 1, - unit.call_id, - unit.user, - unit.assistant - )); - } - body -} - -pub fn evaluation_units( - records: &[crate::EventRecord], - scope: JudgmentScope, -) -> Result> { - let dialogue = dialogue_judge_units(records)?; - Ok(match scope { - JudgmentScope::Story => { - let body = story_judge_body(&dialogue); - if body.trim().is_empty() { - Vec::new() - } else { - vec![EvaluationUnit { - call_id: STORY_CALL_ID.into(), - body, - }] - } - } - JudgmentScope::Turn => dialogue - .into_iter() - .map(|unit| EvaluationUnit { - call_id: unit.call_id, - body: format!("User:\n{}\n\nAssistant:\n{}", unit.user, unit.assistant), - }) - .collect(), - }) -} - -pub fn pending_evaluation_units( - existing: &[JudgeRow], - session_id: &str, - rubric_id: &str, - units: &[EvaluationUnit], - force: bool, -) -> (Vec, usize) { - if force { - return (units.to_vec(), 0); - } - let pending: Vec<_> = units - .iter() - .filter(|unit| !has_judgment(existing, session_id, &unit.call_id, rubric_id)) - .cloned() - .collect(); - let skipped = units.len().saturating_sub(pending.len()); - (pending, skipped) -} - -pub fn manual_judge_rows( - session_id: &str, - rubric_id: &str, - units: &[EvaluationUnit], - inputs: &[ManualJudgmentInput], -) -> Result> { - if inputs.is_empty() { - anyhow::bail!("manual judge requires manual_scores (collect via CLI interactive mode)"); - } - let mut rows = Vec::new(); - for unit in units { - let matches: Vec<_> = inputs - .iter() - .filter(|input| { - input.rubric_id == rubric_id - && input - .call_id - .as_deref() - .filter(|call_id| !call_id.is_empty()) - .unwrap_or(STORY_CALL_ID) - == unit.call_id - }) - .collect(); - if matches.is_empty() { - anyhow::bail!( - "missing manual score for call_id={} rubric={}", - unit.call_id, - rubric_id - ); - } - for input in matches { - rows.push(JudgeRow { - session_id: session_id.into(), - call_id: unit.call_id.clone(), - rubric_id: input.rubric_id.clone(), - score: input.score.clamp(0, 100), - verdict: normalize_verdict(&input.verdict), - rationale: if input.rationale.starts_with(MANUAL_RATIONALE_PREFIX) { - input.rationale.clone() - } else { - format!("{MANUAL_RATIONALE_PREFIX}{}", input.rationale) - }, - }); - } - } - Ok(rows) -} - -pub fn dry_run_judge_rows( - session_id: &str, - rubric_id: &str, - units: &[EvaluationUnit], -) -> Vec { - units - .iter() - .map(|unit| JudgeRow { - session_id: session_id.into(), - call_id: unit.call_id.clone(), - rubric_id: rubric_id.into(), - score: 100, - verdict: "pass".into(), - rationale: "dry-run (no LLM call)".into(), - }) - .collect() -} - -pub fn manual_few_shot_examples( - existing: &[JudgeRow], - rubric_id: &str, - limit: usize, -) -> Vec { - existing - .iter() - .filter(|row| { - row.rubric_id == rubric_id && row.rationale.starts_with(MANUAL_RATIONALE_PREFIX) - }) - .take(limit) - .cloned() - .collect() -} - -pub fn build_llm_judge_prompt( - scope: JudgmentScope, - rubric_id: &str, - units: &[EvaluationUnit], - few_shot: &[JudgeRow], -) -> String { - let mut examples = String::new(); - if !few_shot.is_empty() { - examples.push_str("Reference examples (human scores):\n"); - for example in few_shot { - examples.push_str(&format!( - "- call_id={} score={} verdict={} rationale={}\n", - example.call_id, example.score, example.verdict, example.rationale - )); - } - examples.push('\n'); - } - let trajectory = units - .iter() - .enumerate() - .map(|(index, unit)| match scope { - JudgmentScope::Story => { - format!("### Full trajectory\n{}\n", unit.body) - } - JudgmentScope::Turn => format!( - "### Turn {} (call_id={})\n{}\n", - index + 1, - unit.call_id, - unit.body - ), - }) - .collect::(); - let task = match scope { - JudgmentScope::Story => format!( - "Score the ENTIRE trajectory once (call_id=\"{STORY_CALL_ID}\") on rubric `{rubric_id}`." - ), - JudgmentScope::Turn => { - format!("Score EACH dialogue turn separately on rubric `{rubric_id}`.") - } - }; - format!( - r#"You are an evaluator (LLM-as-judge) for agent trajectories. -{task} -Score 0-100. Verdict: pass, partial, or fail. -Return ONLY valid JSON (no markdown fences): -{{"judgments":[{{"call_id":"...","rubric_id":"{rubric_id}","score":85,"verdict":"pass","rationale":"..."}}]}} - -{examples}Trajectory: -{trajectory}"# - ) -} - -#[derive(serde::Deserialize)] -struct LlmJudgment { - call_id: String, - #[serde(default)] - rubric_id: Option, - score: i64, - verdict: String, - rationale: String, -} - -#[derive(serde::Deserialize)] -struct LlmJudgeBatch { - judgments: Vec, -} - -pub fn parse_llm_judge_rows( - session_id: &str, - rubric_id: &str, - output: &str, -) -> Result> { - let trimmed = output.trim(); - let payload = if trimmed.starts_with("```") { - trimmed - .trim_start_matches("```json") - .trim_start_matches("```") - .trim_end_matches("```") - .trim() - } else { - trimmed - }; - let parsed: LlmJudgeBatch = serde_json::from_str(payload) - .with_context(|| format!("parse judge JSON from model output: {output}"))?; - Ok(parsed - .judgments - .into_iter() - .filter(|judgment| judgment.rubric_id.as_deref().unwrap_or(rubric_id) == rubric_id) - .map(|judgment| JudgeRow { - session_id: session_id.into(), - call_id: judgment.call_id, - rubric_id: rubric_id.into(), - score: judgment.score.clamp(0, 100), - verdict: normalize_verdict(&judgment.verdict), - rationale: judgment.rationale, - }) - .collect()) -} - -fn normalize_verdict(raw: &str) -> String { - match raw.trim().to_ascii_lowercase().as_str() { - "pass" | "ok" | "success" => "pass".into(), - "partial" | "mixed" => "partial".into(), - "fail" | "failed" | "failure" => "fail".into(), - other => other.into(), - } -} - -pub fn has_judgment(rows: &[JudgeRow], session_id: &str, call_id: &str, rubric_id: &str) -> bool { - rows.iter() - .any(|r| r.session_id == session_id && r.call_id == call_id && r.rubric_id == rubric_id) -} - -pub async fn dataset_path(session: &StoryCoords) -> Result { - Ok(story_lance_judgment_path( - &session.storage, - &session.agent_id, - &session.session_id, - session.root_session_id.as_deref(), - )? - .to_string_lossy() - .into_owned()) -} - -const JUDGMENT_SESSION_COL: &str = "session_id"; -const JUDGMENT_CALL_COL: &str = "call_id"; -const JUDGMENT_RUBRIC_COL: &str = "rubric_id"; -const JUDGMENT_SCORE_COL: &str = "score"; -const JUDGMENT_VERDICT_COL: &str = "verdict"; -const JUDGMENT_RATIONALE_COL: &str = "rationale"; - -fn judgment_schema() -> Arc { - Arc::new(ArrowSchema::new(vec![ - Field::new(JUDGMENT_SESSION_COL, DataType::Utf8, false), - Field::new(JUDGMENT_CALL_COL, DataType::Utf8, false), - Field::new(JUDGMENT_RUBRIC_COL, DataType::Utf8, false), - Field::new(JUDGMENT_SCORE_COL, DataType::Int64, false), - Field::new(JUDGMENT_VERDICT_COL, DataType::Utf8, false), - Field::new(JUDGMENT_RATIONALE_COL, DataType::Utf8, false), - ])) -} - -fn judgment_batch(rows: &[JudgeRow]) -> Result { - RecordBatch::try_new( - judgment_schema(), - vec![ - Arc::new(StringArray::from_iter_values( - rows.iter().map(|row| row.session_id.as_str()), - )), - Arc::new(StringArray::from_iter_values( - rows.iter().map(|row| row.call_id.as_str()), - )), - Arc::new(StringArray::from_iter_values( - rows.iter().map(|row| row.rubric_id.as_str()), - )), - Arc::new(Int64Array::from_iter_values( - rows.iter().map(|row| row.score), - )), - Arc::new(StringArray::from_iter_values( - rows.iter().map(|row| row.verdict.as_str()), - )), - Arc::new(StringArray::from_iter_values( - rows.iter().map(|row| row.rationale.as_str()), - )), - ], - ) - .context("build normalized judgment batch") -} - -fn normalized_rows_from_batch(batch: &RecordBatch) -> Result> { - (0..batch.num_rows()) - .map(|index| { - Ok(JudgeRow { - session_id: utf8_at(batch, JUDGMENT_SESSION_COL, index)? - .context("judgment session_id must be non-null")?, - call_id: utf8_at(batch, JUDGMENT_CALL_COL, index)? - .context("judgment call_id must be non-null")?, - rubric_id: utf8_at(batch, JUDGMENT_RUBRIC_COL, index)? - .context("judgment rubric_id must be non-null")?, - score: i64_at(batch, JUDGMENT_SCORE_COL, index)? - .context("judgment score must be non-null")?, - verdict: utf8_at(batch, JUDGMENT_VERDICT_COL, index)? - .context("judgment verdict must be non-null")?, - rationale: utf8_at(batch, JUDGMENT_RATIONALE_COL, index)? - .context("judgment rationale must be non-null")?, - }) - }) - .collect() -} - -/// Read normalized judgments ordered by session, rubric, then unit. -pub async fn read_judge_rows(session: &StoryCoords) -> Result> { - let uri = dataset_path(session).await?; - let ds = match Dataset::open(&uri).await { - Ok(ds) => ds, - Err(lance::Error::DatasetNotFound { .. }) => return Ok(Vec::new()), - Err(error) => return Err(anyhow::anyhow!("{error:#}")).context("open judgments.lance"), - }; - let batches: Vec = ds - .scan() - .try_into_stream() - .await - .context("scan judgments.lance")? - .try_collect() - .await - .context("collect judgments.lance")?; - let mut rows = Vec::new(); - for batch in &batches { - rows.extend(normalized_rows_from_batch(batch)?); - } - rows.sort_by(|a, b| { - a.session_id - .cmp(&b.session_id) - .then_with(|| a.rubric_id.cmp(&b.rubric_id)) - .then_with(|| a.call_id.cmp(&b.call_id)) - }); - Ok(rows) -} - -fn utf8_at(batch: &RecordBatch, name: &str, row: usize) -> Result> { - let Some(idx) = batch.schema().index_of(name).ok() else { - return Ok(None); - }; - let col = batch.column(idx); - let Some(a) = col.as_any().downcast_ref::() else { - anyhow::bail!("expected Utf8 column {name}"); - }; - if a.is_null(row) { - Ok(None) - } else { - Ok(Some(a.value(row).to_string())) - } -} - -fn i64_at(batch: &RecordBatch, name: &str, row: usize) -> Result> { - let Some(idx) = batch.schema().index_of(name).ok() else { - return Ok(None); - }; - let col = batch.column(idx); - let Some(a) = col.as_any().downcast_ref::() else { - anyhow::bail!("expected Int64 column {name}"); - }; - if a.is_null(row) { - Ok(None) - } else { - Ok(Some(a.value(row))) - } -} - -/// Upsert normalized judgments by `(session_id, call_id, rubric_id)`. -pub async fn write_judge_rows(session: &StoryCoords, rows: &[JudgeRow]) -> Result { - if rows.is_empty() { - return dataset_path(session).await; - } - let uri = dataset_path(session).await?; - let _guard = crate::store::dataset_write_lock::acquire(&uri).await?; - - let mut keys = HashSet::new(); - for row in rows { - anyhow::ensure!( - keys.insert((&row.session_id, &row.call_id, &row.rubric_id)), - "duplicate judgment key ({}, {}, {}) in one write", - row.session_id, - row.call_id, - row.rubric_id - ); - } - let batch = judgment_batch(rows)?; - - match Dataset::open(&uri).await { - Ok(ds) => { - let reader = Box::new(RecordBatchIterator::new(vec![Ok(batch)], judgment_schema())); - MergeInsertBuilder::try_new( - Arc::new(ds), - vec![ - JUDGMENT_SESSION_COL.to_string(), - JUDGMENT_CALL_COL.to_string(), - JUDGMENT_RUBRIC_COL.to_string(), - ], - ) - .context("build judgment upsert")? - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .context("build normalized judgment merge job")? - .execute_reader(reader) - .await - .context("upsert judgments.lance")?; - } - Err(lance::Error::DatasetNotFound { .. }) => { - if !uri.contains("://") { - if let Some(parent) = std::path::Path::new(&uri).parent() { - tokio::fs::create_dir_all(parent) - .await - .with_context(|| format!("create judgment root {}", parent.display()))?; - } - } - InsertBuilder::new(&uri) - .execute(vec![batch]) - .await - .with_context(|| format!("create judgments.lance at {uri}"))?; - } - Err(error) => return Err(anyhow::anyhow!("{error:#}")).context("open judgments.lance"), - } - - Ok(uri) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dialogue_units_pair_canonical_events_by_call_id() { - let records = vec![ - crate::EventRecord { - identity: crate::EventIdentity::default(), - seq: 0, - source: "test".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-1".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"user_content": "hello"}), - }, - crate::EventRecord { - identity: crate::EventIdentity::default(), - seq: 1, - source: "test".into(), - kind: "llm.response".into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-1".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"assistant_content": "world"}), - }, - ]; - let units = dialogue_judge_units(&records).unwrap(); - assert_eq!( - units, - vec![JudgeDialogueUnit { - call_id: "call-1".into(), - user: "hello".into(), - assistant: "world".into(), - }] - ); - } -} diff --git a/crates/persisting-pchronicle/src/judgment_summary.rs b/crates/persisting-pchronicle/src/judgment_summary.rs deleted file mode 100644 index 577eb07f..00000000 --- a/crates/persisting-pchronicle/src/judgment_summary.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! Protocol-independent aggregation of persisted judgment columns. - -use std::collections::{HashMap, HashSet}; - -use anyhow::Result; - -use crate::{ - drop_lifecycle_run_partitions, expand_story_locations, judgment_dataset_path, - list_story_read_locations, read_judge_rows, JudgeRow, StoryCoords, MANUAL_RATIONALE_PREFIX, - STORY_CALL_ID, -}; - -#[derive(Debug, Clone, PartialEq)] -pub struct JudgmentSessionSummary { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub root_session_id: Option, - pub judgment_count: usize, - pub turn_judgments: usize, - pub story_judgments: usize, - pub rubric_ids: Vec, - pub avg_score: Option, - pub verdict_pass: usize, - pub verdict_partial: usize, - pub verdict_fail: usize, - pub manual_count: usize, - pub judgments_path: String, - pub status: String, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct JudgmentRubricSummary { - pub rubric_id: String, - pub judgment_count: usize, - pub avg_score: f64, - pub verdict_pass: usize, - pub verdict_partial: usize, - pub verdict_fail: usize, - pub manual_count: usize, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct JudgmentAggregate { - pub storage: String, - pub session_count: usize, - pub judged_session_count: usize, - pub judgment_count: usize, - pub rubric_count: usize, - pub sessions: Vec, - pub rubrics: Vec, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct RunKey { - storage: String, - agent_id: String, - root: String, -} - -fn run_bucket(location: &StoryCoords) -> RunKey { - RunKey { - storage: location.storage.clone(), - agent_id: location.agent_id.clone(), - root: location - .root_session_id - .clone() - .unwrap_or_else(|| location.session_id.clone()), - } -} - -fn run_coords(key: &RunKey) -> StoryCoords { - StoryCoords::new( - &key.storage, - &key.agent_id, - &key.root, - Some(key.root.clone()), - ) -} - -async fn rows_for_run(key: &RunKey) -> Result> { - read_judge_rows(&run_coords(key)).await -} - -pub async fn session_judgment_summary(location: &StoryCoords) -> JudgmentSessionSummary { - let run = run_bucket(location); - let rows = rows_for_run(&run).await.unwrap_or_default(); - let scoped: Vec<_> = rows - .into_iter() - .filter(|row| row.session_id == location.session_id) - .collect(); - session_entry(location, &run, &scoped).await -} - -pub async fn aggregate_judgments( - storage: String, - agent_id: Option, - session_id: Option, - root_session_id: Option, -) -> Result { - let mut locations = list_story_read_locations( - storage.clone(), - agent_id, - session_id.clone(), - root_session_id, - )?; - if session_id.is_none() { - locations = expand_story_locations(locations).await?; - locations = drop_lifecycle_run_partitions(locations); - } - if locations.is_empty() { - anyhow::bail!("judge stats: no sessions found under {storage}"); - } - - let mut rows_by_run: HashMap> = HashMap::new(); - for location in &locations { - rows_by_run.entry(run_bucket(location)).or_default(); - } - for (run, rows) in &mut rows_by_run { - *rows = rows_for_run(run).await?; - } - - let mut all_rows = Vec::new(); - let mut seen = HashSet::new(); - for rows in rows_by_run.values() { - for row in rows { - let key = ( - row.session_id.clone(), - row.call_id.clone(), - row.rubric_id.clone(), - ); - if seen.insert(key) { - all_rows.push(row.clone()); - } - } - } - - let mut sessions = Vec::with_capacity(locations.len()); - for location in &locations { - let run = run_bucket(location); - let scoped: Vec<_> = rows_by_run - .get(&run) - .into_iter() - .flatten() - .filter(|row| row.session_id == location.session_id) - .cloned() - .collect(); - sessions.push(session_entry(location, &run, &scoped).await); - } - - let rubrics = rubric_summaries(&all_rows); - let judged_session_count = sessions - .iter() - .filter(|summary| summary.judgment_count > 0) - .count(); - let session_count = sessions.len(); - let judgment_count = all_rows.len(); - let rubric_count = rubrics.len(); - Ok(JudgmentAggregate { - storage, - session_count, - judged_session_count, - judgment_count, - rubric_count, - sessions, - rubrics, - status: if judgment_count > 0 { "ok" } else { "empty" }.into(), - note: format!( - "Judge stats: {judged_session_count}/{session_count} session(s) with judgments, \ - {judgment_count} judgment(s), {rubric_count} rubric(s)" - ), - }) -} - -async fn session_entry( - location: &StoryCoords, - run: &RunKey, - rows: &[JudgeRow], -) -> JudgmentSessionSummary { - let (verdict_pass, verdict_partial, verdict_fail) = verdict_counts(rows.iter()); - let manual_count = rows - .iter() - .filter(|row| row.rationale.starts_with(MANUAL_RATIONALE_PREFIX)) - .count(); - let turn_judgments = rows - .iter() - .filter(|row| row.call_id != STORY_CALL_ID) - .count(); - JudgmentSessionSummary { - storage: location.storage.clone(), - agent_id: location.agent_id.clone(), - session_id: location.session_id.clone(), - root_session_id: location.root_session_id.clone(), - judgment_count: rows.len(), - turn_judgments, - story_judgments: rows.len().saturating_sub(turn_judgments), - rubric_ids: rubric_ids(rows), - avg_score: average_score(rows.iter()), - verdict_pass, - verdict_partial, - verdict_fail, - manual_count, - judgments_path: judgment_dataset_path(&run_coords(run)) - .await - .unwrap_or_default(), - status: if rows.is_empty() { "empty" } else { "ok" }.into(), - } -} - -fn rubric_summaries(rows: &[JudgeRow]) -> Vec { - rubric_ids(rows) - .into_iter() - .map(|rubric_id| { - let scoped: Vec<_> = rows - .iter() - .filter(|row| row.rubric_id == rubric_id) - .collect(); - let (verdict_pass, verdict_partial, verdict_fail) = - verdict_counts(scoped.iter().copied()); - JudgmentRubricSummary { - rubric_id, - judgment_count: scoped.len(), - avg_score: average_score(scoped.iter().copied()).unwrap_or(0.0), - verdict_pass, - verdict_partial, - verdict_fail, - manual_count: scoped - .iter() - .filter(|row| row.rationale.starts_with(MANUAL_RATIONALE_PREFIX)) - .count(), - } - }) - .collect() -} - -fn rubric_ids(rows: &[JudgeRow]) -> Vec { - let mut ids: Vec<_> = rows - .iter() - .map(|row| row.rubric_id.clone()) - .collect::>() - .into_iter() - .collect(); - ids.sort(); - ids -} - -fn verdict_counts<'a>(rows: impl Iterator) -> (usize, usize, usize) { - let mut counts = (0, 0, 0); - for row in rows { - match row.verdict.as_str() { - "pass" => counts.0 += 1, - "partial" => counts.1 += 1, - "fail" => counts.2 += 1, - _ => {} - } - } - counts -} - -fn average_score<'a>(rows: impl Iterator) -> Option { - let scores: Vec<_> = rows.map(|row| row.score).collect(); - if scores.is_empty() { - None - } else { - Some(scores.iter().sum::() as f64 / scores.len() as f64) - } -} diff --git a/crates/persisting-pchronicle/src/layout/coords.rs b/crates/persisting-pchronicle/src/layout/coords.rs index d4ea06cc..fa21d88e 100644 --- a/crates/persisting-pchronicle/src/layout/coords.rs +++ b/crates/persisting-pchronicle/src/layout/coords.rs @@ -48,15 +48,6 @@ impl StoryCoords { self.root_session_id.as_deref(), ) } - - pub fn lance_judgment_path(&self) -> Result { - story_lance_judgment_path( - &self.storage, - &self.agent_id, - &self.session_id, - self.root_session_id.as_deref(), - ) - } } fn validate_storage(storage: &str) -> Result<()> { @@ -112,20 +103,6 @@ pub fn story_lance_event_path( Ok(run.join("events.lance")) } -/// Normalized judgment dataset at `{run}/judgments.lance/`. -/// -/// Judgments are derived annotations and intentionally do not evolve the -/// canonical `events.lance` schema. -pub fn story_lance_judgment_path( - storage: &str, - agent_id: &str, - session_id: &str, - root_session_id: Option<&str>, -) -> Result { - let run = story_run_dir(storage, agent_id, session_id, root_session_id)?; - Ok(run.join("judgments.lance")) -} - #[cfg(test)] mod tests { use super::*; @@ -153,12 +130,6 @@ mod tests { ); } - #[test] - fn judgments_are_run_scoped_but_physically_separate_from_events() { - let path = story_lance_judgment_path("/store", "agent", "child", Some("run-x")).unwrap(); - assert!(path.ends_with("agent/run-x/judgments.lance")); - } - #[test] fn object_store_uri_preserves_scheme_and_run_partitioning() { let root = story_lance_event_path( diff --git a/crates/persisting-pchronicle/src/layout/mod.rs b/crates/persisting-pchronicle/src/layout/mod.rs index 1e94dcd1..876a20cf 100644 --- a/crates/persisting-pchronicle/src/layout/mod.rs +++ b/crates/persisting-pchronicle/src/layout/mod.rs @@ -4,7 +4,7 @@ mod coords; mod markdown; mod resolve; -pub use coords::{story_lance_event_path, story_lance_judgment_path, story_run_dir, StoryCoords}; +pub use coords::{story_lance_event_path, story_run_dir, StoryCoords}; pub use markdown::{ is_subagent_session_storage_key, is_trajectory_markdown_path, locate_run_bucket_markdown, locate_session_markdown, locate_session_markdown_for_key, sanitize_session_filename, diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index d7f37681..1cdf281c 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -1,7 +1,7 @@ //! pChronicle — Persisting's structured storage layer for Agent trajectories. //! //! pChronicle owns the trajectory formats, physical schemas, storage backends, -//! replay, conversion, search, judgment, and rebuildable views. Capture and +//! replay, conversion, search, and rebuildable views. Capture and //! clients call pChronicle directly; there is no separate storage engine layer. //! //! # Format architecture @@ -29,12 +29,6 @@ pub mod error; pub mod format; pub mod formats; pub mod interop; -#[cfg(feature = "lance-store")] -pub mod judge_service; -#[cfg(feature = "lance-store")] -pub mod judgment; -#[cfg(feature = "lance-store")] -pub mod judgment_summary; pub mod layout; pub mod mapping; mod messages; @@ -94,30 +88,13 @@ pub use formats::{ ActfObservation, ActfStep, ActfToolCall, ActfTrajectory, ACTF_SCHEMA_VERSION, }; pub use interop::{events_to_har, events_to_otlp_json, otlp_json_to_events}; -#[cfg(feature = "lance-store")] -pub use judge_service::{ - judge_trajectory, JudgeTrajectoryOutcome, JudgeTrajectoryRequest, JudgingMethod, -}; -#[cfg(feature = "lance-store")] -pub use judgment::{ - build_llm_judge_prompt, dataset_path as judgment_dataset_path, dialogue_judge_units, - dry_run_judge_rows, evaluation_units, has_judgment, manual_few_shot_examples, - manual_judge_rows, parse_llm_judge_rows, pending_evaluation_units, read_judge_rows, - story_judge_body, write_judge_rows, EvaluationUnit, JudgeDialogueUnit, JudgeRow, JudgmentScope, - ManualJudgmentInput, MANUAL_RATIONALE_PREFIX, STORY_CALL_ID, -}; -#[cfg(feature = "lance-store")] -pub use judgment_summary::{ - aggregate_judgments, session_judgment_summary, JudgmentAggregate, JudgmentRubricSummary, - JudgmentSessionSummary, -}; pub use layout::{ is_subagent_session_storage_key, is_trajectory_markdown_path, list_story_read_locations, locate_run_bucket_markdown, locate_session_markdown, locate_session_markdown_for_key, merge_story_location, resolve_story_read_location, sanitize_session_filename, session_markdown_filename, session_markdown_path_for_key, session_markdown_write_path_for_key, - story_lance_event_path, story_lance_judgment_path, story_run_dir, try_infer_story_location, - StoryCoords, StoryLocationPartial, + story_lance_event_path, story_run_dir, try_infer_story_location, StoryCoords, + StoryLocationPartial, }; pub use mapping::{ agenticmd_block_to_event_record, agenticmd_block_to_replay_json, diff --git a/crates/persisting-pchronicle/src/messages.rs b/crates/persisting-pchronicle/src/messages.rs index 6871027d..9b336c0a 100644 --- a/crates/persisting-pchronicle/src/messages.rs +++ b/crates/persisting-pchronicle/src/messages.rs @@ -337,25 +337,6 @@ pub struct TrajectoryStatsRequest { pub root_session_id: Option, } -/// Judge sidecar summary for one trajectory session (nested in [`TrajectoryStatsResponse`]). -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct SessionJudgeStats { - pub judgment_count: usize, - pub turn_judgments: usize, - pub story_judgments: usize, - pub rubric_ids: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub avg_score: Option, - pub verdict_pass: usize, - pub verdict_partial: usize, - pub verdict_fail: usize, - pub manual_count: usize, - /// Path to the normalized judgment dataset. - #[serde(default, skip_serializing_if = "String::is_empty")] - pub judgments_path: String, - pub status: String, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TrajectoryStatsResponse { pub storage: String, @@ -370,8 +351,6 @@ pub struct TrajectoryStatsResponse { /// only; canonical events remain at-least-once and are never hidden. #[serde(default)] pub duplicate_event_ids: usize, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub judge: Option, pub status: String, pub note: String, } @@ -398,181 +377,6 @@ pub struct TrajectoryMaterializeResponse { pub note: String, } -/// What to score: whole story once, or each dialogue turn. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum JudgeScope { - #[default] - Turn, - Story, -} - -/// LLM-as-judge or human-entered scores (CLI interactive → [`JudgeScoreInput`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum JudgeMethod { - #[default] - Llm, - Manual, -} - -/// How to pick sessions when `--sample` is set (CLI-side). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum JudgeSampleMode { - #[default] - Sequential, - Random, -} - -/// One score dimension entry (manual submit or few-shot export). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct JudgeScoreInput { - /// Turn scope: dialogue `call_id`. Story scope: omit (pChronicle uses `__story__`). - #[serde(default)] - pub call_id: Option, - pub rubric_id: String, - pub score: i64, - pub verdict: String, - pub rationale: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryJudgeRequest { - pub storage: String, - pub agent_id: String, - pub session_id: String, - #[serde(default)] - pub root_session_id: Option, - /// Primary rubric when [`Self::rubric_ids`] is empty. - #[serde(default = "default_judge_rubric")] - pub rubric_id: String, - /// Multiple score dimensions (e.g. `helpful,correct,safe`). - #[serde(default)] - pub rubric_ids: Vec, - #[serde(default)] - pub scope: JudgeScope, - #[serde(default)] - pub method: JudgeMethod, - /// OpenAI-compatible chat model; falls back to `PERSISTING_JUDGE_MODEL` or `gpt-4o-mini`. - #[serde(default)] - pub model: Option, - /// Skip LLM; write deterministic pass rows (for tests / dry runs). - #[serde(default)] - pub dry_run: bool, - /// Re-judge units that already have a row for this rubric. - #[serde(default)] - pub force: bool, - /// Manual method: scores collected interactively by CLI. - #[serde(default)] - pub manual_scores: Vec, - /// LLM method: include up to N prior `[manual]` rows as few-shot examples per rubric. - #[serde(default)] - pub few_shot_limit: usize, -} - -fn default_judge_rubric() -> String { - "default".into() -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryJudgeResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub rubric_id: String, - pub rubric_ids: Vec, - pub scope: JudgeScope, - pub method: JudgeMethod, - /// Path to the normalized `judgments.lance` dataset. - pub judgments_path: String, - /// Units scored this run (turns or one story). - pub judged_calls: usize, - pub skipped_calls: usize, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryJudgeStatsRequest { - pub storage: String, - #[serde(default)] - pub agent_id: Option, - #[serde(default)] - pub session_id: Option, - #[serde(default)] - pub root_session_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct JudgeStatsSession { - pub storage: String, - pub agent_id: String, - pub session_id: String, - #[serde(default)] - pub root_session_id: Option, - pub judgment_count: usize, - pub turn_judgments: usize, - pub story_judgments: usize, - pub rubric_ids: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub avg_score: Option, - pub verdict_pass: usize, - pub verdict_partial: usize, - pub verdict_fail: usize, - pub manual_count: usize, - pub judgments_path: String, - pub status: String, -} - -impl From<&JudgeStatsSession> for SessionJudgeStats { - fn from(s: &JudgeStatsSession) -> Self { - Self { - judgment_count: s.judgment_count, - turn_judgments: s.turn_judgments, - story_judgments: s.story_judgments, - rubric_ids: s.rubric_ids.clone(), - avg_score: s.avg_score, - verdict_pass: s.verdict_pass, - verdict_partial: s.verdict_partial, - verdict_fail: s.verdict_fail, - manual_count: s.manual_count, - judgments_path: s.judgments_path.clone(), - status: s.status.clone(), - } - } -} - -impl From for SessionJudgeStats { - fn from(s: JudgeStatsSession) -> Self { - SessionJudgeStats::from(&s) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct JudgeRubricSummary { - pub rubric_id: String, - pub judgment_count: usize, - pub avg_score: f64, - pub verdict_pass: usize, - pub verdict_partial: usize, - pub verdict_fail: usize, - pub manual_count: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryJudgeStatsResponse { - pub storage: String, - pub session_count: usize, - pub judged_session_count: usize, - pub judgment_count: usize, - pub rubric_count: usize, - pub sessions: Vec, - pub rubrics: Vec, - pub status: String, - pub note: String, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TrajectoryExtractRequest { pub storage: String, diff --git a/crates/persisting-pchronicle/src/operations/trajectory/judge.rs b/crates/persisting-pchronicle/src/operations/trajectory/judge.rs deleted file mode 100644 index efea309e..00000000 --- a/crates/persisting-pchronicle/src/operations/trajectory/judge.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Typed adapter for pChronicle-owned trajectory judgment. - -use crate::{ - judge_trajectory, JudgeTrajectoryRequest, JudgingMethod, JudgmentScope, ManualJudgmentInput, -}; -use crate::{JudgeMethod, JudgeScope, TrajectoryJudgeRequest, TrajectoryJudgeResponse}; -use anyhow::Result; - -pub async fn judge_async(request: TrajectoryJudgeRequest) -> Result { - let session = super::session_from_request( - &request.storage, - &request.agent_id, - &request.session_id, - request.root_session_id.as_deref(), - ); - let scope = match request.scope { - JudgeScope::Story => JudgmentScope::Story, - JudgeScope::Turn => JudgmentScope::Turn, - }; - let method = match request.method { - JudgeMethod::Manual => JudgingMethod::Manual, - JudgeMethod::Llm => JudgingMethod::Llm, - }; - let outcome = judge_trajectory(JudgeTrajectoryRequest { - session, - rubric_id: request.rubric_id, - rubric_ids: request.rubric_ids, - scope, - method, - force: request.force, - dry_run: request.dry_run, - model: request.model, - few_shot_limit: request.few_shot_limit, - manual_scores: request - .manual_scores - .into_iter() - .map(|score| ManualJudgmentInput { - call_id: score.call_id, - rubric_id: score.rubric_id, - score: score.score, - verdict: score.verdict, - rationale: score.rationale, - }) - .collect(), - }) - .await?; - - Ok(TrajectoryJudgeResponse { - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - rubric_id: outcome.primary_rubric, - rubric_ids: outcome.rubric_ids, - scope: request.scope, - method: request.method, - judgments_path: outcome.dataset, - judged_calls: outcome.judged_units, - skipped_calls: outcome.skipped_units, - status: outcome.status, - note: outcome.note, - }) -} diff --git a/crates/persisting-pchronicle/src/operations/trajectory/judge_stats.rs b/crates/persisting-pchronicle/src/operations/trajectory/judge_stats.rs deleted file mode 100644 index 0aeab92a..00000000 --- a/crates/persisting-pchronicle/src/operations/trajectory/judge_stats.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! Typed adapter for pChronicle-owned judgment aggregation. - -use crate::{ - aggregate_judgments, session_judgment_summary, JudgmentRubricSummary, JudgmentSessionSummary, - StoryCoords, -}; -use crate::{ - JudgeRubricSummary, JudgeStatsSession, SessionJudgeStats, TrajectoryJudgeStatsRequest, - TrajectoryJudgeStatsResponse, -}; -use anyhow::Result; - -fn session_to_proto(summary: JudgmentSessionSummary) -> JudgeStatsSession { - JudgeStatsSession { - storage: summary.storage, - agent_id: summary.agent_id, - session_id: summary.session_id, - root_session_id: summary.root_session_id, - judgment_count: summary.judgment_count, - turn_judgments: summary.turn_judgments, - story_judgments: summary.story_judgments, - rubric_ids: summary.rubric_ids, - avg_score: summary.avg_score, - verdict_pass: summary.verdict_pass, - verdict_partial: summary.verdict_partial, - verdict_fail: summary.verdict_fail, - manual_count: summary.manual_count, - judgments_path: summary.judgments_path, - status: summary.status, - } -} - -fn rubric_to_proto(summary: JudgmentRubricSummary) -> JudgeRubricSummary { - JudgeRubricSummary { - rubric_id: summary.rubric_id, - judgment_count: summary.judgment_count, - avg_score: summary.avg_score, - verdict_pass: summary.verdict_pass, - verdict_partial: summary.verdict_partial, - verdict_fail: summary.verdict_fail, - manual_count: summary.manual_count, - } -} - -pub async fn judge_stats_async( - request: TrajectoryJudgeStatsRequest, -) -> Result { - let summary = aggregate_judgments( - request.storage, - request.agent_id, - request.session_id, - request.root_session_id, - ) - .await?; - Ok(TrajectoryJudgeStatsResponse { - storage: summary.storage, - session_count: summary.session_count, - judged_session_count: summary.judged_session_count, - judgment_count: summary.judgment_count, - rubric_count: summary.rubric_count, - sessions: summary.sessions.into_iter().map(session_to_proto).collect(), - rubrics: summary.rubrics.into_iter().map(rubric_to_proto).collect(), - status: summary.status, - note: summary.note, - }) -} - -pub async fn session_judge_stats(location: &StoryCoords) -> SessionJudgeStats { - session_to_proto(session_judgment_summary(location).await).into() -} diff --git a/crates/persisting-pchronicle/src/operations/trajectory/mod.rs b/crates/persisting-pchronicle/src/operations/trajectory/mod.rs index 822b2a37..80d185fd 100644 --- a/crates/persisting-pchronicle/src/operations/trajectory/mod.rs +++ b/crates/persisting-pchronicle/src/operations/trajectory/mod.rs @@ -12,15 +12,12 @@ use crate::{ }; pub use crate::{ TrajectoryAppendRequest, TrajectoryAppendResponse, TrajectoryExtractRequest, - TrajectoryExtractResponse, TrajectoryJudgeRequest, TrajectoryJudgeResponse, - TrajectoryJudgeStatsRequest, TrajectoryJudgeStatsResponse, TrajectoryMaterializeRequest, - TrajectoryMaterializeResponse, TrajectoryReplayRequest, TrajectoryReplayResponse, - TrajectoryStatsRequest, TrajectoryStatsResponse, + TrajectoryExtractResponse, TrajectoryMaterializeRequest, TrajectoryMaterializeResponse, + TrajectoryReplayRequest, TrajectoryReplayResponse, TrajectoryStatsRequest, + TrajectoryStatsResponse, }; use anyhow::Result; -mod judge; -mod judge_stats; fn session_from_request( storage: &str, agent_id: &str, @@ -140,25 +137,20 @@ pub async fn stats_async(request: TrajectoryStatsRequest) -> Result 0 { "ok" } else { "empty" }.into(), - note: format!( - "Canonical Lance event log: {} row(s){projection_note}", - layers.event_rows - ), - }, - &session, - ) - .await) + Ok(TrajectoryStatsResponse { + dataset: layers.event_log_path, + storage: request.storage, + agent_id: request.agent_id, + session_id: request.session_id, + row_count: layers.event_rows, + manifest_revision: RawEventLanceStore.stats(&session).await?.manifest_revision, + duplicate_event_ids, + status: if layers.event_rows > 0 { "ok" } else { "empty" }.into(), + note: format!( + "Canonical Lance event log: {} row(s){projection_note}", + layers.event_rows + ), + }) } async fn duplicate_event_id_count(session: &StoryCoords) -> Result { @@ -175,24 +167,6 @@ async fn duplicate_event_id_count(session: &StoryCoords) -> Result { Ok(counts.values().map(|count| count.saturating_sub(1)).sum()) } -async fn stats_response_with_judge( - mut response: TrajectoryStatsResponse, - session: &StoryCoords, -) -> TrajectoryStatsResponse { - response.judge = Some(judge_stats::session_judge_stats(session).await); - response -} - -pub async fn judge_async(request: TrajectoryJudgeRequest) -> Result { - judge::judge_async(request).await -} - -pub async fn judge_stats_async( - request: TrajectoryJudgeStatsRequest, -) -> Result { - judge_stats::judge_stats_async(request).await -} - pub async fn extract_async(request: TrajectoryExtractRequest) -> Result { let root_session_id = request.root_session_id.as_deref(); let session = session_from_request( diff --git a/crates/persisting-pchronicle/src/revision.rs b/crates/persisting-pchronicle/src/revision.rs index 04f5be7d..1e2b3a61 100644 --- a/crates/persisting-pchronicle/src/revision.rs +++ b/crates/persisting-pchronicle/src/revision.rs @@ -55,25 +55,25 @@ fn schema() -> Arc { fn batch(rows: &[RevisionRow]) -> Result { let strings = |values: Vec| Arc::new(StringArray::from(values)) as _; + let parent_revision_ids = rows + .iter() + .map(|row| serde_json::to_string(&row.parent_revision_ids)) + .collect::>>()?; + let output_refs = rows + .iter() + .map(|row| serde_json::to_string(&row.output_refs)) + .collect::>>()?; RecordBatch::try_new( schema(), vec![ strings(rows.iter().map(|r| r.revision_id.clone()).collect()), - strings( - rows.iter() - .map(|r| serde_json::to_string(&r.parent_revision_ids).unwrap()) - .collect(), - ), + strings(parent_revision_ids), strings(rows.iter().map(|r| r.kind.clone()).collect()), strings(rows.iter().map(|r| r.canonical_snapshot.clone()).collect()), strings(rows.iter().map(|r| r.recipe.to_string()).collect()), strings(rows.iter().map(|r| r.status.clone()).collect()), strings(rows.iter().map(|r| r.created_at.clone()).collect()), - strings( - rows.iter() - .map(|r| serde_json::to_string(&r.output_refs).unwrap()) - .collect(), - ), + strings(output_refs), ], ) .context("build revision catalog batch") diff --git a/crates/persisting-pchronicle/src/store/catalog/discovery.rs b/crates/persisting-pchronicle/src/store/catalog/discovery.rs index 1fb2ec19..edc94293 100644 --- a/crates/persisting-pchronicle/src/store/catalog/discovery.rs +++ b/crates/persisting-pchronicle/src/store/catalog/discovery.rs @@ -512,10 +512,9 @@ fn discover_local_candidates( last_modified: modified_string(&metadata), }); } else if is_lance_directory(&path) { - // Derived Lance datasets such as judgments.lance and - // revisions.lance are sidecars of a canonical Run, not - // trajectory sources. Never descend into their internal - // JSON metadata and register it as an outer file source. + // Derived Lance datasets are sidecars of a canonical Run, + // not trajectory sources. Never descend into their internal + // metadata and register it as an outer file source. } else { pending.push(path); } diff --git a/crates/persisting-pchronicle/src/store/catalog/provider.rs b/crates/persisting-pchronicle/src/store/catalog/provider.rs index b2a08ad1..cab2f0c1 100644 --- a/crates/persisting-pchronicle/src/store/catalog/provider.rs +++ b/crates/persisting-pchronicle/src/store/catalog/provider.rs @@ -155,7 +155,11 @@ impl TableProvider for CatalogTableProvider { let selected_source_count = plans.len(); let plan: Arc = match selected_source_count { 0 => Arc::new(EmptyExec::new(output_schema)), - 1 => plans.pop().expect("one Catalog source plan"), + 1 => plans.pop().ok_or_else(|| { + DataFusionError::Internal( + "Catalog planned one source but produced no execution plan".into(), + ) + })?, _ => UnionExec::try_new(plans)?, }; Ok(match limit { diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index a2033cfa..4545b8db 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -184,10 +184,10 @@ async fn discovers_mixed_local_files_and_exposes_sources() -> Result<()> { #[tokio::test] async fn ignores_derived_lance_sidecars_during_discovery() -> Result<()> { let temp = tempfile::tempdir()?; - fs::create_dir_all(temp.path().join("run/judgments.lance/_versions"))?; + fs::create_dir_all(temp.path().join("run/derived-metrics.lance/_versions"))?; fs::write( temp.path() - .join("run/judgments.lance/_versions/latest_version_hint.json"), + .join("run/derived-metrics.lance/_versions/latest_version_hint.json"), "{}", )?; write_openai_source(&temp.path().join("trajectory.json"), "event-1")?; diff --git a/crates/persisting-pchronicle/src/store/index_build_gate.rs b/crates/persisting-pchronicle/src/store/index_build_gate.rs index dceb3671..8760baf4 100644 --- a/crates/persisting-pchronicle/src/store/index_build_gate.rs +++ b/crates/persisting-pchronicle/src/store/index_build_gate.rs @@ -5,11 +5,10 @@ use std::sync::{Arc, OnceLock}; /// Lance index creation performs external sorts whose merge buffers are large /// relative to the default DataFusion memory pool. Serializing builds keeps /// concurrent Run finalization from multiplying those reservations. -pub(crate) async fn acquire() -> tokio::sync::OwnedSemaphorePermit { - static GATE: OnceLock> = OnceLock::new(); - GATE.get_or_init(|| Arc::new(tokio::sync::Semaphore::new(1))) +pub(crate) async fn acquire() -> tokio::sync::OwnedMutexGuard<()> { + static GATE: OnceLock>> = OnceLock::new(); + GATE.get_or_init(|| Arc::new(tokio::sync::Mutex::new(()))) .clone() - .acquire_owned() + .lock_owned() .await - .expect("pChronicle index-build gate cannot be closed") } diff --git a/crates/persisting-pchronicle/src/store/root_write_lock.rs b/crates/persisting-pchronicle/src/store/root_write_lock.rs index b07979ad..b452663e 100644 --- a/crates/persisting-pchronicle/src/store/root_write_lock.rs +++ b/crates/persisting-pchronicle/src/store/root_write_lock.rs @@ -11,7 +11,9 @@ type WriteLock = tokio::sync::Mutex<()>; pub(super) fn for_root(root: &str) -> Arc { static LOCKS: OnceLock>>> = OnceLock::new(); let locks = LOCKS.get_or_init(|| Mutex::new(HashMap::new())); - let mut locks = locks.lock().expect("pChronicle root lock registry"); + let mut locks = locks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); locks.retain(|_, lock| lock.strong_count() > 0); if let Some(lock) = locks.get(root).and_then(Weak::upgrade) { return lock; diff --git a/crates/persisting-pvisor/src/cli/run.rs b/crates/persisting-pvisor/src/cli/run.rs index 220cddbc..65585021 100644 --- a/crates/persisting-pvisor/src/cli/run.rs +++ b/crates/persisting-pvisor/src/cli/run.rs @@ -27,6 +27,13 @@ use crate::{ use super::trajectory::{chronicle_sink, ChronicleWriter}; +type ChronicleSinks = ( + Arc, + Arc, + Option, + Option>, +); + #[cfg(target_os = "linux")] pub(super) const RUN_COMMAND_ABOUT: &str = "Execute one Agent Run; --safe selects the rootless Linux sandbox"; @@ -853,12 +860,8 @@ async fn execute_config( .dir .clone() .or_else(|| Some(storage.join("chronicle"))); - let (sink, event_sink, writer, chronicle_control): ( - Arc, - Arc, - Option, - Option>, - ) = match config.chronicle.mode { + let (sink, event_sink, writer, chronicle_control): ChronicleSinks = match config.chronicle.mode + { ChronicleMode::Off => ( Arc::new(SeqOnlySink::new()), Arc::new(crate::NoopEventSink), diff --git a/docs/src/pchronicle/concepts/facts-and-projections.md b/docs/src/pchronicle/concepts/facts-and-projections.md index 5657027b..3cf58654 100644 --- a/docs/src/pchronicle/concepts/facts-and-projections.md +++ b/docs/src/pchronicle/concepts/facts-and-projections.md @@ -8,7 +8,7 @@ pChronicle separates what happened from the views used to inspect it. | Logical projection | normalized query model | `runs`, `steps`, `tool_calls`, `trajectories` | | Human projection | readable diagnostic view | AgenticMD | | Exchange representation | interoperability boundary | ATIF, ACTF, OpenAI Messages, Storyline JSON | -| Revision | derived data with lineage | cleaned or augmented trajectories, judgments | +| Revision | derived data with lineage | cleaned, redacted, or augmented trajectories | Canonical events are append-oriented facts. A projection may reorganize those facts for a session or query, but it must not silently become a second source @@ -22,7 +22,7 @@ AgenticMD is a non-authoritative human-readable projection. A missing or stale Markdown view does not change the canonical event result. A revision points to its parent and the transform that produced it. Cleaning, -redaction, augmentation, and judgment therefore create new lineage rather than +redaction, and augmentation therefore create new lineage rather than rewriting history without a trace. Read [Trajectory storage](../design/trajectory-storage.md) for ownership and diff --git a/docs/src/pchronicle/concepts/facts-and-projections.zh.md b/docs/src/pchronicle/concepts/facts-and-projections.zh.md index 12d1a59e..04778cd5 100644 --- a/docs/src/pchronicle/concepts/facts-and-projections.zh.md +++ b/docs/src/pchronicle/concepts/facts-and-projections.zh.md @@ -8,7 +8,7 @@ pChronicle 把“发生了什么”与“如何查看它”分开。 | Logical projection | 规范化查询模型 | `runs`、`steps`、`tool_calls`、`trajectories` | | Human projection | 人读诊断视图 | AgenticMD | | Exchange representation | 互操作边界 | ATIF、ACTF、OpenAI Messages、Storyline JSON | -| Revision | 带 lineage 的派生数据 | 清洗或增广轨迹、Judgment | +| Revision | 带 lineage 的派生数据 | 清洗、脱敏或增广轨迹 | Canonical event 是 append-oriented 的事实。Projection 可以为会话或查询重新组织这些事实， 但不能静默成为第二个事实源。可重建视图需要记录输入 Snapshot 和 transform version。 @@ -18,7 +18,7 @@ Storyline 是会话导向的 projection。它的三表 Lance 布局为完整文 AgenticMD 是非权威的人读 projection。Markdown 视图缺失或过期不会改变 canonical event 结果。 -Revision 指向 parent 和生成它的 transform。清洗、脱敏、增广和评测因此创建新 lineage， +Revision 指向 parent 和生成它的 transform。清洗、脱敏和增广因此创建新 lineage， 而不是无痕改写历史。 所有权见[轨迹存储](../design/trajectory-storage.md)，精确交换契约见 diff --git a/docs/src/pchronicle/design/catalog.md b/docs/src/pchronicle/design/catalog.md index b35b62bc..28edfb76 100644 --- a/docs/src/pchronicle/design/catalog.md +++ b/docs/src/pchronicle/design/catalog.md @@ -416,8 +416,8 @@ Catalog 复用直接文件查询的资源参数： | `POST /api/catalog` | 在锁外完整构建新快照，成功后原子替换，并清空轨迹缓存 | 刷新失败不会清空或部分更新旧 Catalog；正在处理的请求持有旧快照的 `Arc`，可以继续完成。 -Web Explorer 从 Catalog 获取 Dataset 列表，服务端过滤、URL 状态、Storyline 列表和 -judgment key 均携带完整 `(dataset, _file_, session_id)`；`run_id` 作为物理 Run 分组信息 +Web Explorer 从 Catalog 获取 Dataset 列表，服务端过滤、URL 状态和 Storyline 列表 +均携带完整 `(dataset, _file_, session_id)`；`run_id` 作为物理 Run 分组信息 单独返回。Catalog 是不可变快照，新增数据只在显式 refresh 后进入 Web 视图。 ### 9.1 Server source-routing 加速 @@ -454,8 +454,8 @@ value 数，并通过 `failed` 列出本 generation 已缓存的构建失败， ### 9.2 写入边界 -`pchronicle serve` 只提供读取、Catalog 刷新和有界 evidence query，不暴露 judgment 写入、 -maintenance、导入或任意 SQL 写接口。服务强制限制为 loopback；Gateway 和原生 writer +`pchronicle serve` 只提供读取、Catalog 刷新和有界 evidence query，不暴露 maintenance、 +导入或任意 SQL 写接口。服务强制限制为 loopback；Gateway 和原生 writer 直接写 Dataset，不经过 Warehouse API。 ## 10. Rust API 边界 diff --git a/docs/src/pchronicle/design/catalog.zh.md b/docs/src/pchronicle/design/catalog.zh.md index b35b62bc..28edfb76 100644 --- a/docs/src/pchronicle/design/catalog.zh.md +++ b/docs/src/pchronicle/design/catalog.zh.md @@ -416,8 +416,8 @@ Catalog 复用直接文件查询的资源参数： | `POST /api/catalog` | 在锁外完整构建新快照，成功后原子替换，并清空轨迹缓存 | 刷新失败不会清空或部分更新旧 Catalog；正在处理的请求持有旧快照的 `Arc`，可以继续完成。 -Web Explorer 从 Catalog 获取 Dataset 列表，服务端过滤、URL 状态、Storyline 列表和 -judgment key 均携带完整 `(dataset, _file_, session_id)`；`run_id` 作为物理 Run 分组信息 +Web Explorer 从 Catalog 获取 Dataset 列表，服务端过滤、URL 状态和 Storyline 列表 +均携带完整 `(dataset, _file_, session_id)`；`run_id` 作为物理 Run 分组信息 单独返回。Catalog 是不可变快照，新增数据只在显式 refresh 后进入 Web 视图。 ### 9.1 Server source-routing 加速 @@ -454,8 +454,8 @@ value 数，并通过 `failed` 列出本 generation 已缓存的构建失败， ### 9.2 写入边界 -`pchronicle serve` 只提供读取、Catalog 刷新和有界 evidence query，不暴露 judgment 写入、 -maintenance、导入或任意 SQL 写接口。服务强制限制为 loopback；Gateway 和原生 writer +`pchronicle serve` 只提供读取、Catalog 刷新和有界 evidence query，不暴露 maintenance、 +导入或任意 SQL 写接口。服务强制限制为 loopback；Gateway 和原生 writer 直接写 Dataset，不经过 Warehouse API。 ## 10. Rust API 边界 diff --git a/docs/src/pchronicle/design/trajectory-storage.md b/docs/src/pchronicle/design/trajectory-storage.md index 80a0026b..2cbdbc43 100644 --- a/docs/src/pchronicle/design/trajectory-storage.md +++ b/docs/src/pchronicle/design/trajectory-storage.md @@ -13,7 +13,7 @@ - Lance canonical events 的读写、统计和维护； - AgenticMD 人读/调试视图的生成与宽松解析； - events、Storyline、ATIF、ACTF、OpenAI messages、AgenticMD 之间的格式转换； -- materialize、judgment 和标准查询视图。 +- materialize、revision lineage 和标准查询视图。 `persisting-events` 拥有存储无关的逻辑事件信封。Gateway 与 pVisor 负责产出事件；CLI 可以在进程内调用 pChronicle，pVisor 也可以通过 `pchronicle control` sidecar 提交。 @@ -58,9 +58,8 @@ vacuum，避免破坏已经固定旧快照的 reader。 `timestamp` 或接收时间补齐；两者同时存在时必须在毫秒级一致。Storyline 投影也从 `timestamp_unix_ms` 生成 UTC 毫秒文本，输入文本时间戳保存在 `payload_json`。事实层不检查 `event_id` 唯一性，也不为它维护索引； -重复 ID 和重试行是合法事实。完整 `EventRecord` 仍保存在 `payload_json`，因此回放不丢字段。评测结果写入同 Run 的 -`judgments.lance/`，不会随 rubric 增加而演化事实表 schema。需要审计保真度的工作流应 -使用 canonical events 层。 +重复 ID 和重试行是合法事实。完整 `EventRecord` 仍保存在 `payload_json`，因此回放不丢字段。 +需要审计保真度的工作流应使用 canonical events 层。 ### AgenticMD @@ -163,7 +162,7 @@ OpenAI msg ┘ | 组件 | 负责 | 不负责 | |---|---|---| | Gateway | 协议解析、调用生命周期、采集顺序、live projection 策略 | 通用 store、格式 schema、离线转换 | -| pChronicle | 格式、路径、落盘、读取、转换、judgment 与 revision lineage | 网络转发、Agent 生命周期 | +| pChronicle | 格式、路径、落盘、读取、转换与 revision lineage | 网络转发、Agent 生命周期 | | pVisor | Run 生命周期及 Gateway/OverlayNet/OverlayFS 装配 | 长期轨迹 schema | ## 8. 相关文档 diff --git a/docs/src/pchronicle/design/trajectory-storage.zh.md b/docs/src/pchronicle/design/trajectory-storage.zh.md index 80a0026b..2cbdbc43 100644 --- a/docs/src/pchronicle/design/trajectory-storage.zh.md +++ b/docs/src/pchronicle/design/trajectory-storage.zh.md @@ -13,7 +13,7 @@ - Lance canonical events 的读写、统计和维护； - AgenticMD 人读/调试视图的生成与宽松解析； - events、Storyline、ATIF、ACTF、OpenAI messages、AgenticMD 之间的格式转换； -- materialize、judgment 和标准查询视图。 +- materialize、revision lineage 和标准查询视图。 `persisting-events` 拥有存储无关的逻辑事件信封。Gateway 与 pVisor 负责产出事件；CLI 可以在进程内调用 pChronicle，pVisor 也可以通过 `pchronicle control` sidecar 提交。 @@ -58,9 +58,8 @@ vacuum，避免破坏已经固定旧快照的 reader。 `timestamp` 或接收时间补齐；两者同时存在时必须在毫秒级一致。Storyline 投影也从 `timestamp_unix_ms` 生成 UTC 毫秒文本，输入文本时间戳保存在 `payload_json`。事实层不检查 `event_id` 唯一性，也不为它维护索引； -重复 ID 和重试行是合法事实。完整 `EventRecord` 仍保存在 `payload_json`，因此回放不丢字段。评测结果写入同 Run 的 -`judgments.lance/`，不会随 rubric 增加而演化事实表 schema。需要审计保真度的工作流应 -使用 canonical events 层。 +重复 ID 和重试行是合法事实。完整 `EventRecord` 仍保存在 `payload_json`，因此回放不丢字段。 +需要审计保真度的工作流应使用 canonical events 层。 ### AgenticMD @@ -163,7 +162,7 @@ OpenAI msg ┘ | 组件 | 负责 | 不负责 | |---|---|---| | Gateway | 协议解析、调用生命周期、采集顺序、live projection 策略 | 通用 store、格式 schema、离线转换 | -| pChronicle | 格式、路径、落盘、读取、转换、judgment 与 revision lineage | 网络转发、Agent 生命周期 | +| pChronicle | 格式、路径、落盘、读取、转换与 revision lineage | 网络转发、Agent 生命周期 | | pVisor | Run 生命周期及 Gateway/OverlayNet/OverlayFS 装配 | 长期轨迹 schema | ## 8. 相关文档 diff --git a/docs/src/rfcs/0002-events-format.md b/docs/src/rfcs/0002-events-format.md index d7cdc261..81638bfc 100644 --- a/docs/src/rfcs/0002-events-format.md +++ b/docs/src/rfcs/0002-events-format.md @@ -68,7 +68,7 @@ Persisting Gateway 的主入口是代理流量。`events` 应对齐这一现实 | **Replayable** | 在凭证策略允许的前提下，应能从 events 重建「对同一 endpoint 再发一次等价请求」所需信息 | | **Re-derivable views** | Storyline / Markdown / ATIF 视为可从 events **重新投影**的视图 | | **Correlation envelope** | 顶栏保留 `session_id` / `call_id` / `trace_id` 等关联键，不把故事语义塞进 wire | -| **Append-only** | `seq` 单调；不原地改写既有 wire payload（旁路列如 judge 除外） | +| **Append-only** | `seq` 单调；不原地改写既有 wire payload | | **Hub via storyline** | 与其它外围格式互转 MUST 经 storyline；无损路径仍是读 events | 非目标： diff --git a/docs/src/rfcs/0003-pchronicle-ownership.md b/docs/src/rfcs/0003-pchronicle-ownership.md index 9f64884e..72675a4c 100644 --- a/docs/src/rfcs/0003-pchronicle-ownership.md +++ b/docs/src/rfcs/0003-pchronicle-ownership.md @@ -31,7 +31,6 @@ provider/SSE payload 解释可作为 Gateway 扩展行为存在，但不得形 | 人读/调试视图 | `materialize_lance_to_markdown`, AgenticMD 文件 helpers | 从 canonical events 单向生成，可随时删除和重建 | | 发现 | `expand_story_locations` | 发现 canonical Run/Story 分区；Markdown 不参与存储层选择 | | 数据维护 | `RawEventLanceStore::maintain` | 显式离线 compaction、session 索引和 vacuum；事实层不支持 truncate/overwrite | -| judgment 持久化 | `JudgeRow`, `read_judge_rows`, `write_judge_rows` | 独立 `judgments.lance` 的规范化 upsert 及 judge unit 投影 | | Storyline 三表 | `StorylineLanceStore`, `StorylineDataSource` | 原子提交并查询 `runs` / `steps` / `tool_calls` | `events.lance` 是事实源。AgenticMD 和 Storyline 三表均可重建，不可被当作协议级审计或回放的事实源；ATIF 是互操作文档格式，不是独立存储模型。append、replay、stats 不得回退到 AgenticMD。 @@ -100,7 +99,7 @@ Run lease epoch MUST 通过 `EventWriterFence` 进入 canonical event 提交协 ## 验收条件 - Workspace 不再包含 `persisting-engine` crate、动态库、C ABI 或 Engine RPC 信封。 -- append、replay、stats、judge 等轨迹流程均由 pChronicle 提供。 +- append、replay、stats 等轨迹流程均由 pChronicle 提供。 - CLI 不再实现 AgenticMD 到 event 的独立解析。 - Gateway 不再定义与 `EventRecord` 同构的序列化 struct，也不再独立实现 AgenticMD 文档重写或索引。 - pChronicle MUST 使用 Gateway 的真实 AgenticMD、request/response、provider snapshot 与 SSE fixture 验证 wire、Arrow、Lance 和投影兼容性。 diff --git a/docs/src/rfcs/0005-pchronicle-revision-lineage.md b/docs/src/rfcs/0005-pchronicle-revision-lineage.md index 7e5c715b..9019fc73 100644 --- a/docs/src/rfcs/0005-pchronicle-revision-lineage.md +++ b/docs/src/rfcs/0005-pchronicle-revision-lineage.md @@ -9,7 +9,7 @@ ## 摘要 -clean、judge、augment 和格式化数据集是 canonical events 的派生产物。它们不得覆写或 +clean、redact、augment 和格式化数据集是 canonical events 的派生产物。它们不得覆写或 去重事实流，也不得把 catalog 语义塞入 Storyline `extra_json`。每个 Run 使用独立的 `revisions.lance` 记录 lineage；canonical `events.lance` 继续维持 at-least-once、 append-only 契约。 @@ -19,7 +19,7 @@ append-only 契约。 每个 revision 以 `revision_id` 为 upsert key，包含： - `parent_revision_ids`：零个或多个父 revision； -- `kind`：`clean`、`judge`、`augment`、`export` 或扩展 kind； +- `kind`：`clean`、`redact`、`augment`、`export` 或扩展 kind； - `canonical_snapshot`：输入 event manifest revision或 Storyline snapshot； - `recipe`：可重放的程序、版本、参数和输入摘要 JSON； - `status`：`building`、`ready` 或 `failed`； diff --git a/justfile b/justfile index b6b9c999..cdbcd673 100644 --- a/justfile +++ b/justfile @@ -403,8 +403,7 @@ fmt-check-py: # clippy + ruff（不改写） lint: lint-rust lint-py -lint-rust: - cargo clippy --workspace --all-targets --locked +lint-rust: clippy-deny clippy-pchronicle-panics lint-py: uvx ruff check {{ ruff_lint_paths }} @@ -413,9 +412,11 @@ lint-py: lint-py-all: uvx ruff check {{ ruff_paths }} -# 与 Pulsing 同级的严格 clippy（workspace 未清干净前慎用） clippy-deny: - cargo clippy --workspace --all-targets --locked -- -D warnings + cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings + +clippy-pchronicle-panics: + cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used # 兼容旧名 clippy: diff --git a/pchronicle-web/assets/workbench.css b/pchronicle-web/assets/workbench.css index e3a23783..350bac51 100644 --- a/pchronicle-web/assets/workbench.css +++ b/pchronicle-web/assets/workbench.css @@ -1 +1 @@ -.pc2-shell{height:100vh;display:grid;grid-template-columns:56px minmax(0,1fr);background:#f5f7fa;color:#172033}.pc2-main{position:relative;min-width:0;min-height:0;display:flex;flex-direction:column;overflow:hidden}.pc2-global-error{position:absolute;z-index:60;top:12px;left:50%;transform:translateX(-50%);display:flex;align-items:center;gap:10px;max-width:680px;padding:9px 12px;border:1px solid #fecaca;border-radius:9px;background:#fff7f7;color:#991b1b;font-size:11px;box-shadow:0 8px 24px #7f1d1d1a}.pc2-global-error button{margin-left:auto;border:0;background:transparent;color:inherit;font-size:18px;cursor:pointer}.pc2-page,.pc2-detail{min-height:0;display:flex;flex:1;flex-direction:column}.pc2-page{padding:22px 24px 18px;overflow:hidden}.pc2-page-head,.pc2-detail-head{display:flex;align-items:center;justify-content:space-between;gap:18px}.pc2-page-head{margin-bottom:18px}.pc2-page-head h1,.pc2-detail-head h1{margin:2px 0;color:#101828;font-size:22px;line-height:1.2}.pc2-page-head p:not(.eyebrow){margin:5px 0 0;color:#667085;font-size:12px}.pc2-filterbar{display:flex;align-items:center;gap:8px;margin-bottom:12px}.pc2-filterbar select,.pc2-filterbar button,.pc2-filter-search{height:36px;border:1px solid #d7dce3;border-radius:8px;background:#fff;color:#344054;font-size:11px}.pc2-filterbar select{padding:0 30px 0 10px}.pc2-filter-search{min-width:340px;display:flex;align-items:center;gap:7px;padding:0 10px;color:#98a2b3}.pc2-filter-search:focus-within{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb18}.pc2-filter-search input{min-width:0;flex:1;border:0;outline:0;background:transparent;color:#344054}.pc2-sort{padding:0 11px;cursor:pointer}.pc2-result-count{margin-left:auto;color:#667085;font-size:11px}.pc2-table-wrap{min-height:0;flex:1;border:1px solid #dfe3e8;border-radius:11px;background:#fff;overflow:auto;box-shadow:0 1px 2px #10182808}.pc2-run-table{width:100%;border-collapse:collapse;table-layout:fixed}.pc2-run-table th{position:sticky;z-index:2;top:0;padding:10px 12px;border-bottom:1px solid #e4e7ec;background:#f8fafc;color:#667085;text-align:left;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.07em}.pc2-run-table th:first-child{width:26%}.pc2-run-table th:nth-child(2){width:19%}.pc2-run-table th:nth-child(3){width:10%}.pc2-run-table th:nth-child(4),.pc2-run-table th:nth-child(5),.pc2-run-table th:nth-child(6){width:9%}.pc2-run-table td{height:61px;padding:9px 12px;border-bottom:1px solid #eef0f3;color:#475467;font-size:11px;vertical-align:middle}.pc2-run-table tbody tr{cursor:pointer}.pc2-run-table tbody tr:hover,.pc2-run-table tbody tr:focus-visible{outline:0;background:#f7faff;box-shadow:inset 3px 0 #3b82f6}.pc2-run-table td code{display:block;overflow:hidden;color:#667085;font-size:10px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell{min-width:0;display:flex;flex-direction:column;gap:4px}.pc2-session-cell strong{overflow:hidden;color:#1d2939;font-size:11px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell span{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-number{font:600 11px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-table-skeleton td{height:61px;background:linear-gradient(90deg,#fafafa,#f0f3f7,#fafafa);background-size:200% 100%;animation:shimmer 1.4s infinite}.pc2-empty{min-height:180px;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:5px;color:#98a2b3;text-align:center}.pc2-empty strong{color:#475467;font-size:12px}.pc2-empty span{font-size:10px}.pc2-pagination{display:flex;align-items:center;justify-content:flex-end;gap:12px;padding-top:11px;color:#667085;font-size:10px}.pc2-pagination button{padding:6px 9px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;cursor:pointer}.pc2-pagination button:disabled{opacity:.45;cursor:default}.pc2-status{display:inline-flex;align-items:center;gap:5px;padding:3px 7px;border:1px solid #d0d5dd;border-radius:999px;background:#fff;color:#475467;font-size:9px;font-weight:700;text-transform:uppercase}.pc2-status span{width:5px;height:5px;border-radius:50%;background:currentColor}.pc2-status.good{border-color:#bbf7d0;background:#f0fdf4;color:#15803d}.pc2-status.bad{border-color:#fecaca;background:#fff5f5;color:#b42318}.pc2-status.live{border-color:#bfdbfe;background:#eff6ff;color:#1d4ed8}.pc2-detail-head{min-height:78px;padding:11px 20px;border-bottom:1px solid #e4e7ec;background:#fff}.pc2-detail-title{min-width:0;display:flex;align-items:center;gap:13px}.pc2-detail-title>div{min-width:0}.pc2-detail-title p{margin:0;color:#667085;font-size:10px}.pc2-detail-title h1{max-width:700px;overflow:hidden;font-size:17px;text-overflow:ellipsis;white-space:nowrap}.pc2-detail-title h1+div{display:flex;align-items:center;gap:7px}.pc2-detail-title code{color:#667085;font-size:9px}.pc2-back{height:34px;padding:0 10px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;font-size:10px;cursor:pointer}.pc2-head-actions{display:flex;align-items:center;gap:7px}.pc2-metrics{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));margin:12px 20px;border:1px solid #e1e5ea;border-radius:10px;background:#fff;overflow:hidden}.pc2-metric{min-width:0;display:flex;flex-direction:column;gap:3px;padding:10px 12px;border-right:1px solid #eceef1}.pc2-metric:last-child{border-right:0}.pc2-metric span{color:#667085;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.05em}.pc2-metric strong{overflow:hidden;color:#101828;font:600 18px ui-monospace,SFMono-Regular,Menlo,monospace;text-overflow:ellipsis}.pc2-metric small{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-evidence-grid{min-height:0;display:grid;grid-template-columns:minmax(0,1fr) 370px;gap:12px;flex:1;padding:0 20px 18px}.pc2-trace-surface,.pc2-inspector{min-width:0;min-height:0;display:flex;flex-direction:column;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden}.pc2-trace-toolbar,.pc2-inspector-head{min-height:55px;display:flex;align-items:center;justify-content:space-between;gap:10px;padding:8px 11px;border-bottom:1px solid #eceef1}.pc2-trace-toolbar>div:first-child,.pc2-inspector-head>div{display:flex;flex-direction:column;gap:2px}.pc2-trace-toolbar strong,.pc2-inspector-head strong{color:#1d2939;font-size:12px}.pc2-trace-toolbar span,.pc2-inspector-head span{color:#98a2b3;font-size:9px}.pc2-toolbar-controls{display:flex!important;flex-direction:row!important;align-items:center;gap:6px}.pc2-toolbar-controls input,.pc2-toolbar-controls select{height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#344054;font-size:10px}.pc2-toolbar-controls input{width:170px;padding:0 8px}.pc2-toolbar-controls select{padding:0 24px 0 7px}.pc2-icon{width:30px;height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#475467;cursor:pointer}.pc2-segment{display:flex;border:1px solid #d0d5dd;border-radius:6px;overflow:hidden}.pc2-segment button{height:28px;padding:0 8px;border:0;border-right:1px solid #d0d5dd;background:#fff;color:#667085;font-size:9px;cursor:pointer}.pc2-segment button:last-child{border-right:0}.pc2-segment button.active{background:#eff6ff;color:#1d4ed8;font-weight:700}.pc2-turn-list,.pc2-inspector-body{min-height:0;flex:1;overflow:auto;scrollbar-gutter:stable}.pc2-turn-list{padding:11px 14px 24px}.pc2-tree{display:flex;flex-direction:column}.pc2-turn{display:grid;grid-template-columns:24px minmax(0,1fr);gap:7px;padding:0;border:0;background:transparent;text-align:left;cursor:pointer}.pc2-turn-axis{display:flex;flex-direction:column;align-items:center}.pc2-node{z-index:1;width:10px;height:10px;margin-top:14px;border:2px solid #fff;border-radius:50%;background:#94a3b8;box-shadow:0 0 0 1px #cbd5e1}.pc2-node.user{background:#2563eb}.pc2-node.agent{background:#10b981}.pc2-node.system{background:#f59e0b}.pc2-turn-axis i{width:1px;min-height:42px;flex:1;background:#d7dce2}.pc2-turn-card{min-width:0;margin-bottom:7px;padding:9px 10px;border:1px solid #e4e7ec;border-radius:8px;background:#fff}.pc2-turn:hover .pc2-turn-card,.pc2-turn:focus-visible .pc2-turn-card,.pc2-turn.active .pc2-turn-card{border-color:#93c5fd;box-shadow:0 0 0 3px #2563eb12}.pc2-turn:focus-visible{outline:0}.pc2-turn-top,.pc2-turn-meta{display:flex;align-items:center;gap:6px}.pc2-turn-top strong{color:#344054;font-size:10px}.pc2-turn-top code{color:#667085;font-size:9px}.pc2-grow{flex:1}.pc2-role{padding:2px 5px;border-radius:4px;background:#f2f4f7;color:#475467;font-size:8px;font-weight:800;text-transform:uppercase}.pc2-role.user{background:#eff6ff;color:#1d4ed8}.pc2-role.agent{background:#ecfdf5;color:#047857}.pc2-role.system{background:#fffbeb;color:#b45309}.pc2-error-chip{padding:2px 5px;border-radius:4px;background:#fff1f0;color:#b42318!important;font-size:8px}.pc2-turn-card>p{margin:7px 0;color:#475467;font:10px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-turn-meta{flex-wrap:wrap;color:#98a2b3;font-size:8px}.pc2-turn-meta code{margin-left:auto}.pc2-timeline-ruler,.pc2-time-row{display:grid;grid-template-columns:82px minmax(110px,1fr) 70px minmax(150px,1.5fr);gap:9px;align-items:center}.pc2-timeline-ruler{padding:0 7px 8px;color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-timeline-ruler span:nth-child(2){grid-column:2/4}.pc2-time-row{width:100%;min-height:38px;padding:5px 7px;border:0;border-top:1px solid #f0f1f3;background:#fff;color:#475467;text-align:left;cursor:pointer}.pc2-time-row:hover,.pc2-time-row.active{background:#f7faff}.pc2-time-label{display:flex;align-items:center;gap:5px;font-size:9px}.pc2-time-track{height:8px;border-radius:999px;background:#f0f2f5;overflow:hidden}.pc2-time-track i{display:block;height:100%;border-radius:999px;background:#60a5fa}.pc2-time-track i.bad{background:#ef4444}.pc2-time-row>code{color:#667085;font-size:9px}.pc2-time-preview{overflow:hidden;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-inspector-body{padding:10px}.pc2-inspector-facts{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1px;border:1px solid #e4e7ec;border-radius:7px;background:#e4e7ec;overflow:hidden}.pc2-inspector-facts>div{min-width:0;display:flex;flex-direction:column;gap:3px;padding:7px;background:#fff}.pc2-inspector-facts span{color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-inspector-facts code{overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis}.pc2-evidence-block{margin-top:9px;border:1px solid #e4e7ec;border-radius:7px;overflow:hidden}.pc2-evidence-block summary{padding:8px 9px;background:#f8fafc;color:#475467;font-size:9px;font-weight:700;cursor:pointer}.pc2-evidence-block pre{max-height:310px;margin:0;padding:9px;overflow:auto;border-top:1px solid #e4e7ec;background:#fff;color:#344054;font:9px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap;word-break:break-word}.pc2-inspector-empty,.pc2-copilot-empty{min-height:240px;display:flex;flex-direction:column;align-items:center;justify-content:center;padding:24px;color:#98a2b3;text-align:center}.pc2-inspector-empty>span{font-size:25px}.pc2-inspector-empty strong{margin-top:7px;color:#475467;font-size:12px}.pc2-inspector-empty p{max-width:260px;font-size:10px;line-height:1.6}.pc2-inline-loading,.pc2-loading,.pc2-chat-working{display:flex;align-items:center;justify-content:center;gap:7px;color:#667085;font-size:10px}.pc2-inline-loading{padding:10px}.pc2-loading{min-height:100%;flex-direction:column}.pc2-inline-loading .spinner,.pc2-chat-working .spinner{width:13px;height:13px;margin:0}.pc2-judgment-list{margin-top:12px}.pc2-judgment-list h3{font-size:11px}.pc2-judgment-list>div{display:grid;grid-template-columns:1fr auto auto;gap:7px;align-items:center;margin-top:7px;padding:8px;border:1px solid #e4e7ec;border-radius:7px}.pc2-judgment-list strong{font-size:9px}.pc2-judgment-list>div>span{font:600 9px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-judgment-list p{grid-column:1/-1;margin:0;color:#667085;font-size:9px}.pc2-copilot{position:fixed;z-index:70;top:0;right:0;width:min(460px,42vw);height:100vh;display:flex;flex-direction:column;border-left:1px solid #1f2d40;background:#0d1726;color:#e5edf7;box-shadow:-16px 0 44px #1018282b}.pc2-copilot-head{min-height:62px;display:flex;align-items:center;justify-content:space-between;padding:10px 13px;border-bottom:1px solid #223046}.pc2-copilot-head>div:first-child{display:flex;flex-direction:column;gap:3px}.pc2-copilot-head strong{font-size:13px}.pc2-copilot-head span{color:#8291a5;font-size:9px}.pc2-copilot-head button,.pc2-settings header button,.pc2-judgment-drawer header>button{width:30px;height:30px;border:0;border-radius:6px;background:transparent;color:inherit;font-size:17px;cursor:pointer}.pc2-copilot-head button:hover{background:#ffffff0d}.pc2-context-card{margin:11px;padding:10px;border:1px solid #2b3a4f;border-radius:8px;background:#101e30}.pc2-context-card>div{display:flex;justify-content:space-between;gap:8px;margin-bottom:5px;font-size:9px}.pc2-context-card span{color:#8291a5}.pc2-context-card strong{max-width:260px;overflow:hidden;color:#cbd5e1;text-overflow:ellipsis;white-space:nowrap}.pc2-context-card label{display:flex;align-items:center;gap:6px;margin-top:8px;color:#9fb0c4;font-size:9px}.pc2-skill-chips{display:flex;flex-wrap:wrap;gap:5px;padding:0 11px 10px;border-bottom:1px solid #223046}.pc2-skill-chips button{padding:4px 7px;border:1px solid #33465f;border-radius:999px;background:#142236;color:#a9c7ef;font-size:8px;cursor:pointer}.pc2-chat{min-height:0;flex:1;overflow:auto;padding:12px;scrollbar-gutter:stable}.pc2-chat-welcome{display:flex;min-height:220px;flex-direction:column;align-items:center;justify-content:center;color:#718198;text-align:center}.pc2-chat-welcome>span{font-size:30px;color:#60a5fa}.pc2-chat-welcome strong{margin-top:8px;color:#dbeafe;font-size:12px}.pc2-chat-welcome p{max-width:330px;font-size:10px;line-height:1.6}.pc2-message{max-width:92%;margin-bottom:10px;padding:9px 10px;border-radius:10px;font-size:10px;line-height:1.55}.pc2-message.user{margin-left:auto;background:#2563eb;color:#fff}.pc2-message.assistant{border:1px solid #2b3a4f;background:#101e30;color:#d6e2f1}.pc2-action-label{display:inline-block;margin-bottom:5px;color:#60a5fa;font-size:8px;font-weight:800;text-transform:uppercase;letter-spacing:.07em}.pc2-message-text p{margin:0 0 6px;white-space:pre-wrap}.pc2-bullet{display:flex;gap:6px}.pc2-bullet p{flex:1}.pc2-message-sql{margin-top:7px;border:1px solid #33465f;border-radius:6px;overflow:hidden}.pc2-message-sql summary{padding:5px 7px;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-message-sql pre{max-height:180px;margin:0;padding:7px;overflow:auto;border-top:1px solid #33465f;background:#07101f;color:#bfdbfe;font:8px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap}.pc2-truncated{margin-top:7px;padding:5px 6px;border:1px solid #92400e;border-radius:5px;background:#451a0333;color:#fcd34d;font-size:8px}.pc2-citations{display:flex;flex-wrap:wrap;gap:5px;margin-top:7px}.pc2-citations button{padding:3px 6px;border:1px solid #3b82f6;border-radius:5px;background:#1d4ed822;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-composer{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;padding:10px;border-top:1px solid #223046}.pc2-composer textarea{min-height:62px;padding:8px;border:1px solid #33465f;border-radius:7px;outline:0;background:#101e30;color:#e5edf7;font:10px/1.45 inherit;resize:none}.pc2-composer textarea:focus{border-color:#3b82f6}.pc2-composer button{align-self:end}.pc2-modal-backdrop{position:fixed;z-index:90;inset:0;display:flex;align-items:stretch;justify-content:flex-end;background:#10182866;backdrop-filter:blur(2px)}.pc2-modal-backdrop.high{z-index:100;align-items:center;justify-content:center}.pc2-judgment-drawer{width:min(440px,100vw);display:flex;flex-direction:column;background:#fff;color:#344054;box-shadow:-18px 0 48px #10182833}.pc2-judgment-drawer header,.pc2-settings header{display:flex;align-items:center;justify-content:space-between;padding:17px;border-bottom:1px solid #e4e7ec}.pc2-judgment-drawer h2,.pc2-settings h2{margin:2px 0;font-size:17px}.pc2-judgment-drawer header code{color:#667085;font-size:9px}.pc2-judgment-drawer .pc2-form{flex:1}.pc2-form{display:flex;flex-direction:column;gap:13px;padding:17px}.pc2-form label{display:flex;flex-direction:column;gap:5px;color:#475467;font-size:10px;font-weight:600}.pc2-form input,.pc2-form select,.pc2-form textarea{width:100%;padding:8px;border:1px solid #d0d5dd;border-radius:7px;outline:0;background:#fff;color:#344054;font-size:11px}.pc2-form textarea{min-height:130px;resize:vertical}.pc2-form input:focus,.pc2-form select:focus,.pc2-form textarea:focus{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb16}.pc2-form-error{padding:8px;border:1px solid #fecaca;border-radius:7px;background:#fff5f5;color:#b42318;font-size:9px}.pc2-judgment-drawer footer,.pc2-settings footer{display:flex;justify-content:flex-end;gap:7px;padding:13px 17px;border-top:1px solid #e4e7ec}.pc2-settings{width:min(510px,calc(100vw - 30px));border-radius:11px;background:#fff;color:#344054;box-shadow:0 24px 80px #1018283d;overflow:hidden}.pc2-settings-note{margin:15px 17px 0;padding:9px;border:1px solid #bfdbfe;border-radius:7px;background:#eff6ff;color:#1e40af;font-size:9px;line-height:1.55}@media(max-width:1150px){.pc2-metrics{grid-template-columns:repeat(3,1fr)}.pc2-metric:nth-child(3){border-right:0}.pc2-metric:nth-child(-n+3){border-bottom:1px solid #eceef1}.pc2-evidence-grid{grid-template-columns:minmax(480px,1fr) 330px}.pc2-copilot{width:min(500px,55vw)}}@media(max-width:850px){.pc2-page{padding:16px}.pc2-filterbar{flex-wrap:wrap}.pc2-filter-search{min-width:100%;}.pc2-result-count{margin-left:0}.pc2-run-table{min-width:850px}.pc2-detail-head{align-items:flex-start}.pc2-head-actions a{display:none}.pc2-evidence-grid{grid-template-columns:1fr;overflow:auto}.pc2-trace-surface{min-height:520px}.pc2-inspector{min-height:420px}.pc2-copilot{width:calc(100vw - 56px);max-width:none}.pc2-toolbar-controls input{width:120px}} +.pc2-shell{height:100vh;display:grid;grid-template-columns:56px minmax(0,1fr);background:#f5f7fa;color:#172033}.pc2-main{position:relative;min-width:0;min-height:0;display:flex;flex-direction:column;overflow:hidden}.pc2-global-error{position:absolute;z-index:60;top:12px;left:50%;transform:translateX(-50%);display:flex;align-items:center;gap:10px;max-width:680px;padding:9px 12px;border:1px solid #fecaca;border-radius:9px;background:#fff7f7;color:#991b1b;font-size:11px;box-shadow:0 8px 24px #7f1d1d1a}.pc2-global-error button{margin-left:auto;border:0;background:transparent;color:inherit;font-size:18px;cursor:pointer}.pc2-page,.pc2-detail{min-height:0;display:flex;flex:1;flex-direction:column}.pc2-page{padding:22px 24px 18px;overflow:hidden}.pc2-page-head,.pc2-detail-head{display:flex;align-items:center;justify-content:space-between;gap:18px}.pc2-page-head{margin-bottom:18px}.pc2-page-head h1,.pc2-detail-head h1{margin:2px 0;color:#101828;font-size:22px;line-height:1.2}.pc2-page-head p:not(.eyebrow){margin:5px 0 0;color:#667085;font-size:12px}.pc2-filterbar{display:flex;align-items:center;gap:8px;margin-bottom:12px}.pc2-filterbar select,.pc2-filterbar button,.pc2-filter-search{height:36px;border:1px solid #d7dce3;border-radius:8px;background:#fff;color:#344054;font-size:11px}.pc2-filterbar select{padding:0 30px 0 10px}.pc2-filter-search{min-width:340px;display:flex;align-items:center;gap:7px;padding:0 10px;color:#98a2b3}.pc2-filter-search:focus-within{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb18}.pc2-filter-search input{min-width:0;flex:1;border:0;outline:0;background:transparent;color:#344054}.pc2-sort{padding:0 11px;cursor:pointer}.pc2-result-count{margin-left:auto;color:#667085;font-size:11px}.pc2-table-wrap{min-height:0;flex:1;border:1px solid #dfe3e8;border-radius:11px;background:#fff;overflow:auto;box-shadow:0 1px 2px #10182808}.pc2-run-table{width:100%;border-collapse:collapse;table-layout:fixed}.pc2-run-table th{position:sticky;z-index:2;top:0;padding:10px 12px;border-bottom:1px solid #e4e7ec;background:#f8fafc;color:#667085;text-align:left;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.07em}.pc2-run-table th:first-child{width:26%}.pc2-run-table th:nth-child(2){width:19%}.pc2-run-table th:nth-child(3){width:10%}.pc2-run-table th:nth-child(4),.pc2-run-table th:nth-child(5),.pc2-run-table th:nth-child(6){width:9%}.pc2-run-table td{height:61px;padding:9px 12px;border-bottom:1px solid #eef0f3;color:#475467;font-size:11px;vertical-align:middle}.pc2-run-table tbody tr{cursor:pointer}.pc2-run-table tbody tr:hover,.pc2-run-table tbody tr:focus-visible{outline:0;background:#f7faff;box-shadow:inset 3px 0 #3b82f6}.pc2-run-table td code{display:block;overflow:hidden;color:#667085;font-size:10px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell{min-width:0;display:flex;flex-direction:column;gap:4px}.pc2-session-cell strong{overflow:hidden;color:#1d2939;font-size:11px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell span{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-number{font:600 11px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-table-skeleton td{height:61px;background:linear-gradient(90deg,#fafafa,#f0f3f7,#fafafa);background-size:200% 100%;animation:shimmer 1.4s infinite}.pc2-empty{min-height:180px;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:5px;color:#98a2b3;text-align:center}.pc2-empty strong{color:#475467;font-size:12px}.pc2-empty span{font-size:10px}.pc2-pagination{display:flex;align-items:center;justify-content:flex-end;gap:12px;padding-top:11px;color:#667085;font-size:10px}.pc2-pagination button{padding:6px 9px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;cursor:pointer}.pc2-pagination button:disabled{opacity:.45;cursor:default}.pc2-status{display:inline-flex;align-items:center;gap:5px;padding:3px 7px;border:1px solid #d0d5dd;border-radius:999px;background:#fff;color:#475467;font-size:9px;font-weight:700;text-transform:uppercase}.pc2-status span{width:5px;height:5px;border-radius:50%;background:currentColor}.pc2-status.good{border-color:#bbf7d0;background:#f0fdf4;color:#15803d}.pc2-status.bad{border-color:#fecaca;background:#fff5f5;color:#b42318}.pc2-status.live{border-color:#bfdbfe;background:#eff6ff;color:#1d4ed8}.pc2-detail-head{min-height:78px;padding:11px 20px;border-bottom:1px solid #e4e7ec;background:#fff}.pc2-detail-title{min-width:0;display:flex;align-items:center;gap:13px}.pc2-detail-title>div{min-width:0}.pc2-detail-title p{margin:0;color:#667085;font-size:10px}.pc2-detail-title h1{max-width:700px;overflow:hidden;font-size:17px;text-overflow:ellipsis;white-space:nowrap}.pc2-detail-title h1+div{display:flex;align-items:center;gap:7px}.pc2-detail-title code{color:#667085;font-size:9px}.pc2-back{height:34px;padding:0 10px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;font-size:10px;cursor:pointer}.pc2-head-actions{display:flex;align-items:center;gap:7px}.pc2-metrics{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));margin:12px 20px;border:1px solid #e1e5ea;border-radius:10px;background:#fff;overflow:hidden}.pc2-metric{min-width:0;display:flex;flex-direction:column;gap:3px;padding:10px 12px;border-right:1px solid #eceef1}.pc2-metric:last-child{border-right:0}.pc2-metric span{color:#667085;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.05em}.pc2-metric strong{overflow:hidden;color:#101828;font:600 18px ui-monospace,SFMono-Regular,Menlo,monospace;text-overflow:ellipsis}.pc2-metric small{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-evidence-grid{min-height:0;display:grid;grid-template-columns:minmax(0,1fr) 370px;gap:12px;flex:1;padding:0 20px 18px}.pc2-trace-surface,.pc2-inspector{min-width:0;min-height:0;display:flex;flex-direction:column;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden}.pc2-trace-toolbar,.pc2-inspector-head{min-height:55px;display:flex;align-items:center;justify-content:space-between;gap:10px;padding:8px 11px;border-bottom:1px solid #eceef1}.pc2-trace-toolbar>div:first-child,.pc2-inspector-head>div{display:flex;flex-direction:column;gap:2px}.pc2-trace-toolbar strong,.pc2-inspector-head strong{color:#1d2939;font-size:12px}.pc2-trace-toolbar span,.pc2-inspector-head span{color:#98a2b3;font-size:9px}.pc2-toolbar-controls{display:flex!important;flex-direction:row!important;align-items:center;gap:6px}.pc2-toolbar-controls input,.pc2-toolbar-controls select{height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#344054;font-size:10px}.pc2-toolbar-controls input{width:170px;padding:0 8px}.pc2-toolbar-controls select{padding:0 24px 0 7px}.pc2-icon{width:30px;height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#475467;cursor:pointer}.pc2-segment{display:flex;border:1px solid #d0d5dd;border-radius:6px;overflow:hidden}.pc2-segment button{height:28px;padding:0 8px;border:0;border-right:1px solid #d0d5dd;background:#fff;color:#667085;font-size:9px;cursor:pointer}.pc2-segment button:last-child{border-right:0}.pc2-segment button.active{background:#eff6ff;color:#1d4ed8;font-weight:700}.pc2-turn-list,.pc2-inspector-body{min-height:0;flex:1;overflow:auto;scrollbar-gutter:stable}.pc2-turn-list{padding:11px 14px 24px}.pc2-tree{display:flex;flex-direction:column}.pc2-turn{display:grid;grid-template-columns:24px minmax(0,1fr);gap:7px;padding:0;border:0;background:transparent;text-align:left;cursor:pointer}.pc2-turn-axis{display:flex;flex-direction:column;align-items:center}.pc2-node{z-index:1;width:10px;height:10px;margin-top:14px;border:2px solid #fff;border-radius:50%;background:#94a3b8;box-shadow:0 0 0 1px #cbd5e1}.pc2-node.user{background:#2563eb}.pc2-node.agent{background:#10b981}.pc2-node.system{background:#f59e0b}.pc2-turn-axis i{width:1px;min-height:42px;flex:1;background:#d7dce2}.pc2-turn-card{min-width:0;margin-bottom:7px;padding:9px 10px;border:1px solid #e4e7ec;border-radius:8px;background:#fff}.pc2-turn:hover .pc2-turn-card,.pc2-turn:focus-visible .pc2-turn-card,.pc2-turn.active .pc2-turn-card{border-color:#93c5fd;box-shadow:0 0 0 3px #2563eb12}.pc2-turn:focus-visible{outline:0}.pc2-turn-top,.pc2-turn-meta{display:flex;align-items:center;gap:6px}.pc2-turn-top strong{color:#344054;font-size:10px}.pc2-turn-top code{color:#667085;font-size:9px}.pc2-grow{flex:1}.pc2-role{padding:2px 5px;border-radius:4px;background:#f2f4f7;color:#475467;font-size:8px;font-weight:800;text-transform:uppercase}.pc2-role.user{background:#eff6ff;color:#1d4ed8}.pc2-role.agent{background:#ecfdf5;color:#047857}.pc2-role.system{background:#fffbeb;color:#b45309}.pc2-error-chip{padding:2px 5px;border-radius:4px;background:#fff1f0;color:#b42318!important;font-size:8px}.pc2-turn-card>p{margin:7px 0;color:#475467;font:10px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-turn-meta{flex-wrap:wrap;color:#98a2b3;font-size:8px}.pc2-turn-meta code{margin-left:auto}.pc2-timeline-ruler,.pc2-time-row{display:grid;grid-template-columns:82px minmax(110px,1fr) 70px minmax(150px,1.5fr);gap:9px;align-items:center}.pc2-timeline-ruler{padding:0 7px 8px;color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-timeline-ruler span:nth-child(2){grid-column:2/4}.pc2-time-row{width:100%;min-height:38px;padding:5px 7px;border:0;border-top:1px solid #f0f1f3;background:#fff;color:#475467;text-align:left;cursor:pointer}.pc2-time-row:hover,.pc2-time-row.active{background:#f7faff}.pc2-time-label{display:flex;align-items:center;gap:5px;font-size:9px}.pc2-time-track{height:8px;border-radius:999px;background:#f0f2f5;overflow:hidden}.pc2-time-track i{display:block;height:100%;border-radius:999px;background:#60a5fa}.pc2-time-track i.bad{background:#ef4444}.pc2-time-row>code{color:#667085;font-size:9px}.pc2-time-preview{overflow:hidden;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-inspector-body{padding:10px}.pc2-inspector-facts{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1px;border:1px solid #e4e7ec;border-radius:7px;background:#e4e7ec;overflow:hidden}.pc2-inspector-facts>div{min-width:0;display:flex;flex-direction:column;gap:3px;padding:7px;background:#fff}.pc2-inspector-facts span{color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-inspector-facts code{overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis}.pc2-evidence-block{margin-top:9px;border:1px solid #e4e7ec;border-radius:7px;overflow:hidden}.pc2-evidence-block summary{padding:8px 9px;background:#f8fafc;color:#475467;font-size:9px;font-weight:700;cursor:pointer}.pc2-evidence-block pre{max-height:310px;margin:0;padding:9px;overflow:auto;border-top:1px solid #e4e7ec;background:#fff;color:#344054;font:9px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap;word-break:break-word}.pc2-inspector-empty,.pc2-copilot-empty{min-height:240px;display:flex;flex-direction:column;align-items:center;justify-content:center;padding:24px;color:#98a2b3;text-align:center}.pc2-inspector-empty>span{font-size:25px}.pc2-inspector-empty strong{margin-top:7px;color:#475467;font-size:12px}.pc2-inspector-empty p{max-width:260px;font-size:10px;line-height:1.6}.pc2-inline-loading,.pc2-loading,.pc2-chat-working{display:flex;align-items:center;justify-content:center;gap:7px;color:#667085;font-size:10px}.pc2-inline-loading{padding:10px}.pc2-loading{min-height:100%;flex-direction:column}.pc2-inline-loading .spinner,.pc2-chat-working .spinner{width:13px;height:13px;margin:0}.pc2-copilot{position:fixed;z-index:70;top:0;right:0;width:min(460px,42vw);height:100vh;display:flex;flex-direction:column;border-left:1px solid #1f2d40;background:#0d1726;color:#e5edf7;box-shadow:-16px 0 44px #1018282b}.pc2-copilot-head{min-height:62px;display:flex;align-items:center;justify-content:space-between;padding:10px 13px;border-bottom:1px solid #223046}.pc2-copilot-head>div:first-child{display:flex;flex-direction:column;gap:3px}.pc2-copilot-head strong{font-size:13px}.pc2-copilot-head span{color:#8291a5;font-size:9px}.pc2-copilot-head button,.pc2-settings header button{width:30px;height:30px;border:0;border-radius:6px;background:transparent;color:inherit;font-size:17px;cursor:pointer}.pc2-copilot-head button:hover{background:#ffffff0d}.pc2-context-card{margin:11px;padding:10px;border:1px solid #2b3a4f;border-radius:8px;background:#101e30}.pc2-context-card>div{display:flex;justify-content:space-between;gap:8px;margin-bottom:5px;font-size:9px}.pc2-context-card span{color:#8291a5}.pc2-context-card strong{max-width:260px;overflow:hidden;color:#cbd5e1;text-overflow:ellipsis;white-space:nowrap}.pc2-context-card label{display:flex;align-items:center;gap:6px;margin-top:8px;color:#9fb0c4;font-size:9px}.pc2-skill-chips{display:flex;flex-wrap:wrap;gap:5px;padding:0 11px 10px;border-bottom:1px solid #223046}.pc2-skill-chips button{padding:4px 7px;border:1px solid #33465f;border-radius:999px;background:#142236;color:#a9c7ef;font-size:8px;cursor:pointer}.pc2-chat{min-height:0;flex:1;overflow:auto;padding:12px;scrollbar-gutter:stable}.pc2-chat-welcome{display:flex;min-height:220px;flex-direction:column;align-items:center;justify-content:center;color:#718198;text-align:center}.pc2-chat-welcome>span{font-size:30px;color:#60a5fa}.pc2-chat-welcome strong{margin-top:8px;color:#dbeafe;font-size:12px}.pc2-chat-welcome p{max-width:330px;font-size:10px;line-height:1.6}.pc2-message{max-width:92%;margin-bottom:10px;padding:9px 10px;border-radius:10px;font-size:10px;line-height:1.55}.pc2-message.user{margin-left:auto;background:#2563eb;color:#fff}.pc2-message.assistant{border:1px solid #2b3a4f;background:#101e30;color:#d6e2f1}.pc2-action-label{display:inline-block;margin-bottom:5px;color:#60a5fa;font-size:8px;font-weight:800;text-transform:uppercase;letter-spacing:.07em}.pc2-message-text p{margin:0 0 6px;white-space:pre-wrap}.pc2-bullet{display:flex;gap:6px}.pc2-bullet p{flex:1}.pc2-message-sql{margin-top:7px;border:1px solid #33465f;border-radius:6px;overflow:hidden}.pc2-message-sql summary{padding:5px 7px;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-message-sql pre{max-height:180px;margin:0;padding:7px;overflow:auto;border-top:1px solid #33465f;background:#07101f;color:#bfdbfe;font:8px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap}.pc2-truncated{margin-top:7px;padding:5px 6px;border:1px solid #92400e;border-radius:5px;background:#451a0333;color:#fcd34d;font-size:8px}.pc2-citations{display:flex;flex-wrap:wrap;gap:5px;margin-top:7px}.pc2-citations button{padding:3px 6px;border:1px solid #3b82f6;border-radius:5px;background:#1d4ed822;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-composer{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;padding:10px;border-top:1px solid #223046}.pc2-composer textarea{min-height:62px;padding:8px;border:1px solid #33465f;border-radius:7px;outline:0;background:#101e30;color:#e5edf7;font:10px/1.45 inherit;resize:none}.pc2-composer textarea:focus{border-color:#3b82f6}.pc2-composer button{align-self:end}.pc2-modal-backdrop{position:fixed;z-index:90;inset:0;display:flex;align-items:stretch;justify-content:flex-end;background:#10182866;backdrop-filter:blur(2px)}.pc2-modal-backdrop.high{z-index:100;align-items:center;justify-content:center}.pc2-settings header{display:flex;align-items:center;justify-content:space-between;padding:17px;border-bottom:1px solid #e4e7ec}.pc2-settings h2{margin:2px 0;font-size:17px}.pc2-form{display:flex;flex-direction:column;gap:13px;padding:17px}.pc2-form label{display:flex;flex-direction:column;gap:5px;color:#475467;font-size:10px;font-weight:600}.pc2-form input,.pc2-form select,.pc2-form textarea{width:100%;padding:8px;border:1px solid #d0d5dd;border-radius:7px;outline:0;background:#fff;color:#344054;font-size:11px}.pc2-form textarea{min-height:130px;resize:vertical}.pc2-form input:focus,.pc2-form select:focus,.pc2-form textarea:focus{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb16}.pc2-form-error{padding:8px;border:1px solid #fecaca;border-radius:7px;background:#fff5f5;color:#b42318;font-size:9px}.pc2-settings footer{display:flex;justify-content:flex-end;gap:7px;padding:13px 17px;border-top:1px solid #e4e7ec}.pc2-settings{width:min(510px,calc(100vw - 30px));border-radius:11px;background:#fff;color:#344054;box-shadow:0 24px 80px #1018283d;overflow:hidden}.pc2-settings-note{margin:15px 17px 0;padding:9px;border:1px solid #bfdbfe;border-radius:7px;background:#eff6ff;color:#1e40af;font-size:9px;line-height:1.55}@media(max-width:1150px){.pc2-metrics{grid-template-columns:repeat(3,1fr)}.pc2-metric:nth-child(3){border-right:0}.pc2-metric:nth-child(-n+3){border-bottom:1px solid #eceef1}.pc2-evidence-grid{grid-template-columns:minmax(480px,1fr) 330px}.pc2-copilot{width:min(500px,55vw)}}@media(max-width:850px){.pc2-page{padding:16px}.pc2-filterbar{flex-wrap:wrap}.pc2-filter-search{min-width:100%;}.pc2-result-count{margin-left:0}.pc2-run-table{min-width:850px}.pc2-detail-head{align-items:flex-start}.pc2-head-actions a{display:none}.pc2-evidence-grid{grid-template-columns:1fr;overflow:auto}.pc2-trace-surface{min-height:520px}.pc2-inspector{min-height:420px}.pc2-copilot{width:calc(100vw - 56px);max-width:none}.pc2-toolbar-controls input{width:120px}} diff --git a/pchronicle-web/src/agent.rs b/pchronicle-web/src/agent.rs index 9e179726..20b473f3 100644 --- a/pchronicle-web/src/agent.rs +++ b/pchronicle-web/src/agent.rs @@ -6,7 +6,7 @@ use serde_json::{json, Value}; use crate::api; use crate::components::{table_fence, trajectory_fence}; -use crate::model::{Judgment, RunAnalysis, RunSummary, TurnDetail, TurnSummary}; +use crate::model::{RunAnalysis, RunSummary, TurnDetail, TurnSummary}; const STORAGE_KEY: &str = "pchronicle_llm_config"; const DEFAULT_CONTEXT_LIMIT: usize = 32 * 1024; @@ -52,7 +52,6 @@ pub struct AnswerRequest<'a> { pub analysis: &'a RunAnalysis, pub turns: &'a [TurnSummary], pub selected: Option<&'a TurnDetail>, - pub judgments: &'a [Judgment], pub include_full_turn: bool, } @@ -99,7 +98,6 @@ pub fn skill_ids() -> &'static [&'static str] { "latency_hotspots", "tool_usage", "cohort_compare", - "judgment_review", ] } @@ -111,15 +109,14 @@ pub async fn answer(request: AnswerRequest<'_>) -> Result { analysis, turns, selected, - judgments, include_full_turn, } = request; let (base_context, context_truncated) = - evidence_context(run, analysis, turns, selected, judgments, include_full_turn); + evidence_context(run, analysis, turns, selected, include_full_turn); let explicit_skill = resolve_skill(user_message); if !config.is_configured() { let skill = explicit_skill.unwrap_or("trajectory_summary"); - let (evidence, sql) = run_skill(skill, run, analysis, turns, judgments).await?; + let (evidence, sql) = run_skill(skill, run, analysis, turns).await?; let evidence = decorate_skill_evidence(skill, evidence, turns); return Ok(AgentAnswer { text: format!( @@ -182,7 +179,7 @@ pub async fn answer(request: AnswerRequest<'_>) -> Result { .as_deref() .filter(|skill| skill_ids().contains(skill)) .unwrap_or("trajectory_summary"); - let (evidence, sql) = run_skill(skill, run, analysis, turns, judgments).await?; + let (evidence, sql) = run_skill(skill, run, analysis, turns).await?; let evidence = decorate_skill_evidence(skill, evidence, turns); let components = component_fences(&evidence); let summary = summarize(config, user_message, &base_context, &evidence).await?; @@ -205,7 +202,6 @@ async fn run_skill( run: &RunSummary, analysis: &RunAnalysis, turns: &[TurnSummary], - judgments: &[Judgment], ) -> Result<(String, Option), String> { match skill { "failure_locator" => { @@ -304,23 +300,6 @@ async fn run_skill( Some(sql), )) } - "judgment_review" => Ok(( - if judgments.is_empty() { - "No persisted judgments exist for this session.".into() - } else { - judgments - .iter() - .map(|row| { - format!( - "- target={} rubric={} score={} verdict={} rationale={}", - row.call_id, row.rubric_id, row.score, row.verdict, row.rationale - ) - }) - .collect::>() - .join("\n") - }, - None, - )), _ => Ok((overview_evidence(run, analysis, turns), None)), } } @@ -389,7 +368,7 @@ fn overview_evidence(run: &RunSummary, analysis: &RunAnalysis, turns: &[TurnSumm .collect::>() .join("\n"); format!( - "Run: agent={} session={} status={}\nEvents={} turns={} tools={} explicit_errors={}\nTokens: prompt={} completion={} total={}\nLatency: samples={}/{} p50={} p95={} max={}\nJudgments={} average_score={}\nTurn evidence:\n{}", + "Run: agent={} session={} status={}\nEvents={} turns={} tools={} explicit_errors={}\nTokens: prompt={} completion={} total={}\nLatency: samples={}/{} p50={} p95={} max={}\nTurn evidence:\n{}", run.agent_id, run.session_id, run.status, @@ -405,8 +384,6 @@ fn overview_evidence(run: &RunSummary, analysis: &RunAnalysis, turns: &[TurnSumm optional_number(analysis.latency_ms.p50), optional_number(analysis.latency_ms.p95), optional_number(analysis.latency_ms.max), - analysis.judgment_count, - optional_number(analysis.average_score), top ) } @@ -416,19 +393,9 @@ fn evidence_context( analysis: &RunAnalysis, turns: &[TurnSummary], selected: Option<&TurnDetail>, - judgments: &[Judgment], include_full_turn: bool, ) -> (String, bool) { let mut context = overview_evidence(run, analysis, turns); - if !judgments.is_empty() { - context.push_str("\n\nPersisted judgments:\n"); - for row in judgments.iter().take(20) { - context.push_str(&format!( - "- target={} rubric={} score={} verdict={} rationale={}\n", - row.call_id, row.rubric_id, row.score, row.verdict, row.rationale - )); - } - } if let Some(detail) = selected { context.push_str(&format!( "\nSelected [turn:{}]: source={} kind={} model={} latency={} tools={}\n", @@ -485,7 +452,7 @@ async fn select_action( .map(|catalog| catalog.database.as_str()) .unwrap_or("data"); let system = format!( - "You are pChronicle Copilot for local agent trajectory debugging. Select exactly one action. Return JSON only: {{\"action\":\"skill|sql|answer\",\"skill_id\":\"trajectory_summary|failure_locator|latency_hotspots|tool_usage|cohort_compare|judgment_review|null\",\"sql\":null,\"reply\":\"\"}}. For SQL, emit exactly one read-only SELECT/WITH/EXPLAIN over {database}.runs, {database}.steps, {database}.tool_calls, or {database}.trajectories. Prefer a built-in skill. Never claim missing data is zero and never infer an error from arbitrary message text.\n\nWorkspace evidence:\n{context}" + "You are pChronicle Copilot for local agent trajectory debugging. Select exactly one action. Return JSON only: {{\"action\":\"skill|sql|answer\",\"skill_id\":\"trajectory_summary|failure_locator|latency_hotspots|tool_usage|cohort_compare|null\",\"sql\":null,\"reply\":\"\"}}. For SQL, emit exactly one read-only SELECT/WITH/EXPLAIN over {database}.runs, {database}.steps, {database}.tool_calls, or {database}.trajectories. Prefer a built-in skill. Never claim missing data is zero and never infer an error from arbitrary message text.\n\nWorkspace evidence:\n{context}" ); let text = chat(config, &system, user_message, true).await?; serde_json::from_str(extract_json(&text)) @@ -559,9 +526,6 @@ fn resolve_skill(message: &str) -> Option<&'static str> { "latency_hotspots" => normalized.contains("slow") || normalized.contains("latency"), "tool_usage" => normalized.contains("tool"), "cohort_compare" => normalized.contains("compare") || normalized.contains("cohort"), - "judgment_review" => { - normalized.contains("judgment") || normalized.contains("score") - } _ => false, } }) @@ -573,7 +537,6 @@ fn skill_title(skill: &str) -> &'static str { "latency_hotspots" => "Latency hotspots", "tool_usage" => "Tool usage", "cohort_compare" => "Cohort compare", - "judgment_review" => "Judgment review", _ => "Trajectory summary", } } diff --git a/pchronicle-web/src/api.rs b/pchronicle-web/src/api.rs index 37d01bfb..921699a9 100644 --- a/pchronicle-web/src/api.rs +++ b/pchronicle-web/src/api.rs @@ -1,5 +1,5 @@ use crate::model::{ - Judgment, QueryCatalog, QueryEvidence, RunAnalysis, RunPage, RunSummary, TurnDetail, TurnPage, + QueryCatalog, QueryEvidence, RunAnalysis, RunPage, RunSummary, TurnDetail, TurnPage, }; use gloo_net::http::{Request, Response}; use serde_json::json; @@ -85,19 +85,6 @@ pub async fn turn_detail(run: &RunSummary, turn_id: i64) -> Result Result, String> { - checked( - Request::get(&format!("/api/v1/judgments?{}", run.query())) - .send() - .await - .map_err(|e| e.to_string())?, - ) - .await? - .json() - .await - .map_err(|e| e.to_string()) -} - pub async fn query_evidence(sql: &str) -> Result { query_evidence_with_budget(sql, 50, 64 * 1024).await } diff --git a/pchronicle-web/src/components.rs b/pchronicle-web/src/components.rs index 77cf97fa..159a8dc2 100644 --- a/pchronicle-web/src/components.rs +++ b/pchronicle-web/src/components.rs @@ -426,7 +426,6 @@ fn InlineTurnDetail(value: TurnDetail) -> Element { if !value.wire_tool_calls.is_empty() { EvidenceBlock { title: "Tool calls", value: serde_json::to_string_pretty(&value.wire_tool_calls).unwrap_or_default() } } if let Some(observation) = &value.turn.observation { EvidenceBlock { title: "Observation", value: serde_json::to_string_pretty(observation).unwrap_or_default() } } if !value.events.is_empty() { EvidenceBlock { title: "Raw linked events", value: serde_json::to_string_pretty(&value.events).unwrap_or_default() } } - if !value.judgments.is_empty() { div { class: "pc2-judgment-list", h3 { "Judgments" } for row in value.judgments { div { strong { "{row.rubric_id}" } span { class: "pc2-status {status_tone(&row.verdict)}", span {} "{row.verdict}" } span { "{row.score}/100" } p { "{row.rationale}" } } } } } } } @@ -452,15 +451,6 @@ fn optional_u64(value: Option) -> String { .map(|value| value.to_string()) .unwrap_or_else(|| "—".into()) } -fn status_tone(value: &str) -> &'static str { - match value { - "completed" | "ok" | "pass" => "good", - "failed" | "error" | "fail" => "bad", - "active" => "live", - _ => "neutral", - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/pchronicle-web/src/model.rs b/pchronicle-web/src/model.rs index 4548198a..7459cb48 100644 --- a/pchronicle-web/src/model.rs +++ b/pchronicle-web/src/model.rs @@ -27,9 +27,6 @@ pub struct RunExplorerItem { #[serde(flatten)] pub run: RunSummary, pub model: Option, - pub judgment_count: usize, - pub average_score: Option, - pub verdict: Option, } #[derive(Clone, Debug, PartialEq, Eq, Deserialize)] @@ -233,8 +230,6 @@ pub struct RunAnalysis { pub kind_breakdown: Vec, pub model_breakdown: Vec, pub tools: Vec, - pub judgment_count: usize, - pub average_score: Option, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] @@ -254,7 +249,6 @@ pub struct TurnSummary { pub tool_names: Vec, pub event_seqs: Vec, pub has_error: bool, - pub judgment_count: usize, } #[derive(Clone, Debug, PartialEq, Deserialize)] @@ -263,23 +257,12 @@ pub struct TurnPage { pub records: Vec, } -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -pub struct Judgment { - pub session_id: String, - pub call_id: String, - pub rubric_id: String, - pub score: i64, - pub verdict: String, - pub rationale: String, -} - #[derive(Clone, Debug, PartialEq, Deserialize)] pub struct TurnDetail { pub summary: TurnSummary, pub turn: StorylineTurn, pub wire_tool_calls: Vec, pub events: Vec, - pub judgments: Vec, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] diff --git a/pchronicle-web/src/workspace.rs b/pchronicle-web/src/workspace.rs index ab77c79f..c0b2bb38 100644 --- a/pchronicle-web/src/workspace.rs +++ b/pchronicle-web/src/workspace.rs @@ -7,7 +7,7 @@ use crate::agent::{self, AgentAnswer, LlmConfig}; use crate::api; use crate::components::{parse_rich_blocks, DataTable, RichBlock, TrajectoryView}; use crate::model::{ - DimensionAggregate, HistogramBucket, Judgment, QueryCatalog, QueryDatasetSummary, RunAnalysis, + DimensionAggregate, HistogramBucket, QueryCatalog, QueryDatasetSummary, RunAnalysis, RunExplorerItem, RunPage, RunSummary, ToolAggregate, TurnDetail, TurnSummary, }; @@ -78,7 +78,6 @@ pub fn App() -> Element { let mut selected_run = use_signal(move || initial_run); let mut analysis = use_signal(|| None::); let mut turns = use_signal(Vec::::new); - let mut judgments = use_signal(Vec::::new); let mut selected_turn = use_signal(|| None::); let mut expanded_turn_id = use_signal(|| url_param("turn").and_then(|value| value.parse::().ok())); @@ -113,7 +112,7 @@ pub fn App() -> Element { use_effect(move || { if analysis().is_none() { if let Some(run) = selected_run() { - load_workspace(run, analysis, turns, judgments, detail_loading, error); + load_workspace(run, analysis, turns, detail_loading, error); } } }); @@ -208,14 +207,13 @@ pub fn App() -> Element { rsx! { div { class: "pc2-detail-layout", PathExplorer { runs: path_runs, selected_path, loading: runs_loading(), filter_folders: false, on_path: move |value| { run_path.set(value); offset.set(0); page.set("runs".into()); }, - on_select: move |run: RunSummary| { selected_run.set(Some(run)); analysis.set(None); turns.set(Vec::new()); judgments.set(Vec::new()); selected_turn.set(None); expanded_turn_id.set(None); }, + on_select: move |run: RunSummary| { selected_run.set(Some(run)); analysis.set(None); turns.set(Vec::new()); selected_turn.set(None); expanded_turn_id.set(None); }, } if let (Some(_run), Some(value)) = (selected_run(), analysis()) { RunDetailWorkspace { run: value.run.clone(), analysis: value, turns: turns(), - judgments: judgments(), selected: selected_turn(), expanded_turn_id: expanded_turn_id(), loading: detail_loading(), @@ -257,7 +255,7 @@ pub fn App() -> Element { rsx! { div { class: "pc2-runs-layout", PathExplorer { runs: path_runs, selected_path: run_path(), loading: runs_loading(), filter_folders: true, on_path: move |value| { run_path.set(value); offset.set(0); }, - on_select: move |run: RunSummary| { selected_run.set(Some(run)); analysis.set(None); turns.set(Vec::new()); judgments.set(Vec::new()); selected_turn.set(None); expanded_turn_id.set(None); detail_mode.set("trace".into()); page.set("detail".into()); }, + on_select: move |run: RunSummary| { selected_run.set(Some(run)); analysis.set(None); turns.set(Vec::new()); selected_turn.set(None); expanded_turn_id.set(None); detail_mode.set("trace".into()); page.set("detail".into()); }, } RunsExplorer { page: runs(), @@ -301,7 +299,6 @@ pub fn App() -> Element { selected_run.set(Some(run.clone())); analysis.set(None); turns.set(Vec::new()); - judgments.set(Vec::new()); selected_turn.set(None); expanded_turn_id.set(None); detail_mode.set("trace".into()); @@ -320,7 +317,6 @@ pub fn App() -> Element { analysis: value, turns: turns(), selected: selected_turn(), - judgments: judgments(), on_close: move |_| copilot_open.set(false), on_turn: move |id| { if let Some(run) = selected_run() { @@ -373,29 +369,19 @@ fn load_workspace( run: RunSummary, mut analysis: Signal>, mut turns: Signal>, - mut judgments: Signal>, mut loading: Signal, mut error: Signal>, ) { loading.set(true); spawn(async move { - let (next_analysis, next_turns, next_judgments) = futures_util::join!( - api::run_analysis(&run), - api::turns(&run, "", "all"), - api::judgments(&run), - ); - match (next_analysis, next_turns, next_judgments) { - (Ok(next_analysis), Ok(next_turns), Ok(next_judgments)) => { + let (next_analysis, next_turns) = + futures_util::join!(api::run_analysis(&run), api::turns(&run, "", "all"),); + match (next_analysis, next_turns) { + (Ok(next_analysis), Ok(next_turns)) => { analysis.set(Some(next_analysis)); turns.set(next_turns.records); - judgments.set( - next_judgments - .into_iter() - .filter(|row| row.session_id == run.session_id) - .collect(), - ); } - (Err(message), _, _) | (_, Err(message), _) | (_, _, Err(message)) => { + (Err(message), _) | (_, Err(message)) => { error.set(Some(message)); } } @@ -584,7 +570,7 @@ fn RunsExplorer( rsx! { section { class: "pc2-page", header { class: "pc2-page-head", - div { p { class: "eyebrow", "pChronicle" } h1 { "Trajectory runs" } p { "Inspect agent execution, captured latency, explicit failures, and human judgments." } } + div { p { class: "eyebrow", "pChronicle" } h1 { "Trajectory runs" } p { "Inspect agent execution, captured latency, and explicit failures." } } button { class: "button", onclick: on_refresh, "↻ Refresh" } } div { class: "pc2-filterbar", @@ -594,19 +580,19 @@ fn RunsExplorer( for mounted in datasets { option { value: "{mounted.name}", "{mounted.name}" } } } select { value: "{status}", aria_label: "Filter by run status", onchange: move |event| on_status.call(event.value()), option { value: "all", "All statuses" } option { value: "active", "Active" } option { value: "completed", "Completed" } option { value: "failed", "Failed" } } - select { value: "{sort}", aria_label: "Sort runs", onchange: move |event| on_sort.call(event.value()), option { value: "session", "Session" } option { value: "events", "Events" } option { value: "score", "Score" } option { value: "status", "Status" } option { value: "agent", "Agent" } } + select { value: "{sort}", aria_label: "Sort runs", onchange: move |event| on_sort.call(event.value()), option { value: "session", "Session" } option { value: "events", "Events" } option { value: "status", "Status" } option { value: "agent", "Agent" } } button { class: "pc2-sort", aria_label: "Toggle sort direction", onclick: move |_| on_direction.call(if direction == "asc" { "desc".into() } else { "asc".into() }), if direction == "asc" { "↑ Asc" } else { "↓ Desc" } } if !path.is_empty() { button { class: "pc2-path-filter", title: "{path}", onclick: move |_| on_path.call(String::new()), "⌁ {short(&path, 24)} ×" } } span { class: "pc2-result-count", "{total} runs" } } div { class: "pc2-table-wrap", table { class: "pc2-run-table", - thead { tr { th { "Session" } th { "Agent / model" } th { "Status" } th { "Events" } th { "Judgments" } th { "Score" } th { "Root" } } } + thead { tr { th { "Session" } th { "Agent / model" } th { "Status" } th { "Events" } th { "Root" } } } tbody { if loading && page.is_none() { - for _ in 0..6 { tr { class: "pc2-table-skeleton", td { colspan: "7" } } } + for _ in 0..6 { tr { class: "pc2-table-skeleton", td { colspan: "5" } } } } else if page.as_ref().is_none_or(|page| page.records.is_empty()) { - tr { td { colspan: "7", div { class: "pc2-empty", strong { "No matching trajectories" } span { "Adjust the filters or refresh the local store." } } } } + tr { td { colspan: "5", div { class: "pc2-empty", strong { "No matching trajectories" } span { "Adjust the filters or refresh the local store." } } } } } else { for item in page.as_ref().unwrap().records.iter() { RunTableRow { key: "{item.run.dataset}/{item.run.file}/{item.run.session_id}", item: item.clone(), on_select } @@ -630,10 +616,6 @@ fn RunsExplorer( fn RunTableRow(item: RunExplorerItem, on_select: EventHandler) -> Element { let run = item.run.clone(); let keyboard_run = item.run.clone(); - let score_text = item - .average_score - .map(|value| format!("{value:.1}")) - .unwrap_or_else(|| "—".into()); let root_text = short(item.run.root_session_id.as_deref().unwrap_or("—"), 18); let model_text = item.model.clone().unwrap_or_else(|| "unavailable".into()); rsx! { @@ -642,8 +624,6 @@ fn RunTableRow(item: RunExplorerItem, on_select: EventHandler) -> El td { div { class: "pc2-session-cell", strong { "{item.run.agent_id}" } span { "{model_text}" } } } td { StatusBadge { value: item.run.status.clone() } } td { class: "pc2-number", "{item.run.row_count}" } - td { class: "pc2-number", "{item.judgment_count}" } - td { class: "pc2-number", "{score_text}" } td { code { title: "{item.run.root_session_id.as_deref().unwrap_or_default()}", "{root_text}" } } } } @@ -666,7 +646,6 @@ fn RunDetailWorkspace( run: RunSummary, analysis: RunAnalysis, turns: Vec, - judgments: Vec, selected: Option, expanded_turn_id: Option, loading: bool, @@ -698,7 +677,6 @@ fn RunDetailWorkspace( AnalysisWorkspace { analysis: analysis.clone(), turns: turns.clone(), - judgments, on_turn: move |id| { on_turn.call(id); on_detail_mode.call("trace".into()); @@ -733,7 +711,6 @@ fn MetricsStrip(analysis: RunAnalysis) -> Element { Metric { label: "Explicit errors", value: analysis.error_count.to_string(), detail: "Captured signals only" } Metric { label: "Tokens", value: analysis.total_tokens.map(|value| value.to_string()).unwrap_or_else(|| "—".into()), detail: format!("in {} · out {}", optional_u64(analysis.prompt_tokens), optional_u64(analysis.completion_tokens)) } Metric { label: "Latency P95", value: analysis.latency_ms.p95.map(format_ms).unwrap_or_else(|| "—".into()), detail: format!("{}/{} samples", analysis.latency_ms.sample_count, analysis.latency_ms.total_count) } - Metric { label: "Score", value: analysis.average_score.map(|value| format!("{value:.1}")).unwrap_or_else(|| "—".into()), detail: format!("{} judgments", analysis.judgment_count) } } } } @@ -746,7 +723,6 @@ fn Metric(label: String, value: String, detail: String) -> Element { fn AnalysisWorkspace( analysis: RunAnalysis, turns: Vec, - judgments: Vec, on_turn: EventHandler, ) -> Element { let mut tab = use_signal(|| "overview".to_string()); @@ -757,14 +733,12 @@ fn AnalysisWorkspace( AnalysisTab { value: "performance", label: "Performance", active: active.clone(), on_select: move |value| tab.set(value) } AnalysisTab { value: "tokens", label: "Tokens", active: active.clone(), on_select: move |value| tab.set(value) } AnalysisTab { value: "tools", label: "Tools", active: active.clone(), on_select: move |value| tab.set(value) } - AnalysisTab { value: "quality", label: "Quality", active: active.clone(), on_select: move |value| tab.set(value) } } div { class: "pc2-analysis-scroll", match active.as_str() { "performance" => rsx! { PerformanceAnalysis { analysis: analysis.clone(), turns: turns.clone(), on_turn } }, "tokens" => rsx! { TokenAnalysis { analysis: analysis.clone(), turns: turns.clone(), on_turn } }, "tools" => rsx! { ToolAnalysis { tools: analysis.tools.clone() } }, - "quality" => rsx! { QualityAnalysis { analysis: analysis.clone(), judgments } }, _ => rsx! { OverviewAnalysis { analysis, turns } }, } } @@ -814,7 +788,6 @@ fn OverviewAnalysis(analysis: RunAnalysis, turns: Vec) -> Element { span { strong { "{analysis.models.len()}" } " models" } span { strong { "{analysis.event_count}" } " events" } span { strong { "{analysis.error_count}" } " explicit-error turns" } - span { strong { "{analysis.judgment_count}" } " judgments" } } } } } @@ -926,33 +899,6 @@ fn ToolAnalysis(tools: Vec) -> Element { } } } -#[component] -fn QualityAnalysis(analysis: RunAnalysis, judgments: Vec) -> Element { - let verdicts = verdict_counts(&judgments); - let rubrics = rubric_summaries(&judgments); - rsx! { div { class: "pc2-analysis-grid quality", - AnalysisCard { title: "Explicit errors by source", subtitle: "Captured signals only; message text is not guessed", - ErrorDimensionBars { items: analysis.source_breakdown.clone() } - } - AnalysisCard { title: "Judgment verdicts", subtitle: "Run- and turn-level human evaluation", - div { class: "pc2-verdict-grid", - VerdictCount { label: "Pass", count: verdicts.0, tone: "pass" } - VerdictCount { label: "Partial", count: verdicts.1, tone: "partial" } - VerdictCount { label: "Fail", count: verdicts.2, tone: "fail" } - } - } - article { class: "pc2-analysis-card wide", - header { div { h3 { "Rubric scores" } p { "Average and range across saved judgments" } } } - div { class: "pc2-rubric-list", - if rubrics.is_empty() { EmptyAnalysis { label: "No judgments have been saved" } } - for rubric in rubrics { - div { div { strong { "{rubric.name}" } span { "{rubric.count} judgments · {rubric.min}–{rubric.max}" } } span { class: "pc2-score-track", i { style: format!("width:{}%", rubric.average.clamp(0.0, 100.0)) } } code { "{rubric.average:.1}" } } - } - } - } - } } -} - #[component] fn AnalysisCard(title: &'static str, subtitle: &'static str, children: Element) -> Element { rsx! { article { class: "pc2-analysis-card", header { div { h3 { "{title}" } p { "{subtitle}" } } } {children} } } @@ -992,21 +938,6 @@ fn TokenDimensionBars(items: Vec) -> Element { } } } -#[component] -fn ErrorDimensionBars(items: Vec) -> Element { - let max = items.iter().map(|item| item.error_count).max().unwrap_or(0); - rsx! { div { class: "pc2-dimension-list red", - if max == 0 { EmptyAnalysis { label: "No explicit errors captured" } } - for item in items.into_iter().filter(|item| item.error_count > 0) { - div { class: "pc2-dimension-row", - div { span { "{item.name}" } code { "{item.error_count}" } } - span { class: "pc2-dimension-track", i { style: format!("width:{}%", percent(item.error_count as f64, max as f64)) } } - small { {format!("{:.1}% of this source", percent(item.error_count as f64, item.turn_count as f64))} } - } - } - } } -} - #[component] fn CoverageRow(label: &'static str, observed: usize, total: usize) -> Element { let coverage = percent(observed as f64, total as f64); @@ -1060,32 +991,17 @@ fn TurnMetricChart( } } } -#[component] -fn VerdictCount(label: &'static str, count: usize, tone: &'static str) -> Element { - rsx! { div { class: "pc2-verdict {tone}", span {} strong { "{count}" } small { "{label}" } } } -} - #[component] fn EmptyAnalysis(label: &'static str) -> Element { rsx! { div { class: "pc2-analysis-empty", "{label}" } } } -#[derive(Clone)] -struct RubricSummary { - name: String, - count: usize, - average: f64, - min: i64, - max: i64, -} - #[component] fn CopilotPanel( run: RunSummary, analysis: RunAnalysis, turns: Vec, selected: Option, - judgments: Vec, on_close: EventHandler, on_turn: EventHandler, ) -> Element { @@ -1100,7 +1016,7 @@ fn CopilotPanel( div { class: "pc2-context-card", div { span { "Grounded in" } strong { "{short(&run.session_id, 30)}" } } div { span { "Evidence" } strong { "{analysis.turn_count} turns · {analysis.error_count} explicit errors" } } label { input { r#type: "checkbox", checked: include_full(), disabled: selected.is_none(), onchange: move |event| include_full.set(event.checked()) } "Include selected turn content once (max 64 KiB)" } } div { class: "pc2-skill-chips", for skill in agent::skill_ids() { button { disabled: busy(), onclick: move |_| input.set(format!("/{skill}")), "{skill_label(skill)}" } } } div { class: "pc2-chat", - if messages().is_empty() { div { class: "pc2-chat-welcome", span { "◇" } strong { "Ask from captured evidence" } p { "Copilot can summarize this run, locate explicit failures, rank latency, inspect tool usage, compare cohorts, or review judgments." } } } + if messages().is_empty() { div { class: "pc2-chat-welcome", span { "◇" } strong { "Ask from captured evidence" } p { "Copilot can summarize this run, locate explicit failures, rank latency, inspect tool usage, or compare cohorts." } } } for (index, message) in messages().iter().enumerate() { ChatBubble { key: "message-{index}", message: message.clone(), turns: turns.clone(), on_turn } } if busy() { div { class: "pc2-chat-working", span { class: "spinner" } "Selecting one read-only analysis action…" } } } @@ -1116,7 +1032,6 @@ fn CopilotPanel( let analysis_value = analysis.clone(); let turns_value = turns.clone(); let selected_value = selected.clone(); - let judgments_value = judgments.clone(); let include_value = include_full(); spawn(async move { let result = agent::answer(agent::AnswerRequest { @@ -1126,7 +1041,6 @@ fn CopilotPanel( analysis: &analysis_value, turns: &turns_value, selected: selected_value.as_ref(), - judgments: &judgments_value, include_full_turn: include_value, }).await; let message = match result { @@ -1249,36 +1163,6 @@ fn metric_value(turn: &TurnSummary, metric: &str) -> String { .unwrap_or_else(|| "no latency sample".into()), } } -fn verdict_counts(judgments: &[Judgment]) -> (usize, usize, usize) { - judgments.iter().fold((0, 0, 0), |mut counts, row| { - match row.verdict.as_str() { - "pass" => counts.0 += 1, - "partial" => counts.1 += 1, - "fail" => counts.2 += 1, - _ => {} - } - counts - }) -} -fn rubric_summaries(judgments: &[Judgment]) -> Vec { - let mut grouped = BTreeMap::>::new(); - for judgment in judgments { - grouped - .entry(judgment.rubric_id.clone()) - .or_default() - .push(judgment.score); - } - grouped - .into_iter() - .map(|(name, scores)| RubricSummary { - name, - count: scores.len(), - average: scores.iter().sum::() as f64 / scores.len() as f64, - min: scores.iter().copied().min().unwrap_or_default(), - max: scores.iter().copied().max().unwrap_or_default(), - }) - .collect() -} fn clean_markdown(value: &str) -> String { value.replace("**", "").replace('`', "") } @@ -1420,31 +1304,4 @@ mod tests { assert_eq!(percent(10.0, 0.0), 0.0); assert_eq!(percent(150.0, 100.0), 100.0); } - - #[test] - fn judgment_dimensions_preserve_verdicts_and_rubrics() { - let judgments = vec![ - Judgment { - session_id: "s".into(), - call_id: "a".into(), - rubric_id: "quality".into(), - score: 80, - verdict: "pass".into(), - rationale: "ok".into(), - }, - Judgment { - session_id: "s".into(), - call_id: "b".into(), - rubric_id: "quality".into(), - score: 40, - verdict: "fail".into(), - rationale: "bad".into(), - }, - ]; - assert_eq!(verdict_counts(&judgments), (1, 0, 1)); - let rubrics = rubric_summaries(&judgments); - assert_eq!(rubrics.len(), 1); - assert_eq!(rubrics[0].average, 60.0); - assert_eq!((rubrics[0].min, rubrics[0].max), (40, 80)); - } } From ae988799c43a2329d1d9fffd608da6d57b9fe695 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:02:21 +0800 Subject: [PATCH 05/65] docs: design pchronicle post-judge cleanup --- ...17-pchronicle-post-judge-cleanup-design.md | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md new file mode 100644 index 00000000..98cbf1cf --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md @@ -0,0 +1,74 @@ +# pChronicle Post-Judge Cleanup Design + +## Goal + +Finish the bounded cleanup selected after removing pChronicle's judge subsystem: +remove orphaned Web CSS, replace the remaining production `unreachable!` calls +with explicit errors, and verify supported non-search feature combinations in +local and CI lint gates. + +## Scope + +This change includes only: + +- pChronicle Web CSS class selectors with no static consumer anywhere under + `pchronicle-web/src`; +- the four production `unreachable!` sites currently reported by + `clippy::unreachable` in default-feature pChronicle builds; +- pChronicle feature checks for core-only, local Lance, default S3, and OSS + builds; +- the local `just` lint entry point and the matching GitHub Actions lint step. + +Search, TTAS, queues and samplers, and `persisting-dlcapt` remain out of scope. +The source-only `pc2-token-composition` class is also out of scope because it is +not an orphaned CSS selector and removing or styling it would be a UI decision. + +## Web CSS Cleanup + +The cleanup is conservative and token-based. A `.pc2-*` selector is removable +only when its class token does not occur in any Rust source under +`pchronicle-web/src`. This removes the four definite judge remnants +(`pc2-verdict-grid`, `pc2-verdict`, `pc2-rubric-list`, and `pc2-score-track`) +and the other selectors left behind by superseded Explorer layouts. + +No retained selector is renamed, no layout value is changed, and no source +markup is changed. Verification repeats the cross-file class-token comparison +and runs the Web test suite and build checks. + +## Explicit Error Semantics + +Each remaining production `unreachable!` becomes an ordinary error at the +same abstraction boundary: + +- projection sync reports incompatible non-canonical lineage; +- local manifest construction rejects an unsupported query format; +- Catalog projection binding reports a source-kind inconsistency; +- OpenAI corpus recovery rejects an unknown retained document kind. + +The successful path remains unchanged. After conversion, the pChronicle panic +lint also denies `clippy::unreachable`, alongside `unwrap_used` and +`expect_used`. + +## Feature Matrix + +The local lint entry point and CI verify these non-search configurations: + +1. `--no-default-features` for the lightweight format/event surface; +2. `--no-default-features --features lance-store` for local storage; +3. default features for the S3-compatible product build; +4. `--no-default-features --features oss-store` for the OSS backend. + +Each variant builds the library with warnings denied and the production panic +lints enabled. The workspace-wide strict Clippy command remains the first gate. +CI invokes the same `just lint-rust` recipe used locally, avoiding two copies of +the feature policy. + +## Verification + +- The CSS/source class-token difference contains no CSS-only `.pc2-*` class. +- `clippy::unwrap_used`, `clippy::expect_used`, and `clippy::unreachable` are + clean for all four supported pChronicle feature combinations. +- Workspace strict Clippy remains green with `persisting-dlcapt` excluded. +- pChronicle library, pChronicle CLI, and pChronicle Web tests remain green. +- Existing user-owned untracked review and RFC files remain untouched. + From 59dfaf0f81c0c116d2697ca6bdf6e5a041f7b549 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:06:31 +0800 Subject: [PATCH 06/65] docs: expand pchronicle cleanup design --- ...17-pchronicle-post-judge-cleanup-design.md | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md index 98cbf1cf..92016855 100644 --- a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md +++ b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md @@ -2,10 +2,10 @@ ## Goal -Finish the bounded cleanup selected after removing pChronicle's judge subsystem: +Finish the cleanup selected after removing pChronicle's judge subsystem: remove orphaned Web CSS, replace the remaining production `unreachable!` calls -with explicit errors, and verify supported non-search feature combinations in -local and CI lint gates. +with explicit errors, verify supported non-search feature combinations, and +co-locate the fragmented AgenticMD and Storyline row-model implementations. ## Scope @@ -17,7 +17,10 @@ This change includes only: `clippy::unreachable` in default-feature pChronicle builds; - pChronicle feature checks for core-only, local Lance, default S3, and OSS builds; -- the local `just` lint entry point and the matching GitHub Actions lint step. +- the local `just` lint entry point and the matching GitHub Actions lint step; +- the AgenticMD codecs, mapping, layout, file operations, conversion, and + projection code currently spread across six crate areas; +- the Storyline logical row model currently separated from its Arrow codecs. Search, TTAS, queues and samplers, and `persisting-dlcapt` remain out of scope. The source-only `pc2-token-composition` class is also out of scope because it is @@ -63,6 +66,56 @@ lints enabled. The workspace-wide strict Clippy command remains the first gate. CI invokes the same `just lint-rust` recipe used locally, avoiding two copies of the feature policy. +## AgenticMD Domain Consolidation + +AgenticMD currently spans roughly 2,465 lines across `formats`, `mapping`, +`layout`, `store`, `convert`, and `projection`. The implementation moves into a +single private root subtree with responsibility-oriented files: + +```text +agenticmd/ +├── mod.rs +├── codec.rs +├── body.rs +├── frontmatter.rs +├── validate.rs +├── mapping/ +│ ├── mod.rs +│ ├── fields.rs +│ └── text.rs +├── layout.rs +├── fs.rs +├── convert.rs +└── projection.rs +``` + +`projection.rs` remains gated by `lance-store`. The move changes no wire +format, path rule, mapping rule, filesystem behavior, or projection behavior. +The crate root continues to re-export the existing public AgenticMD functions, +constants, and types, so current Gateway and CLI consumers remain source +compatible. + +Old implementation-oriented module paths such as +`persisting_pchronicle::formats::agenticmd::*` are intentionally not retained +through compatibility wrappers. No workspace consumer uses those paths, the +crate remains pre-1.0, and retaining them would preserve the structure this +change is meant to remove. `formats`, `convert`, `projection`, and `store` may +re-export domain functions at their existing module root where that requires no +wrapper module, but they no longer own AgenticMD implementation files. + +## Storyline Row-Model Consolidation + +The logical Storyline three-table model moves from root-level +`storyline_schema.rs` to `store/storyline/model.rs`. It remains separate from +`store/storyline/rows.rs`: `model.rs` owns normalization, reconstruction, and +logical row types, while `rows.rs` owns Arrow schemas and codecs. Co-location, +not a thousand-line file merge, is the goal. + +The crate root continues to export `split_storyline`, `reconstruct_storyline`, +`StoryRunRow`, `StoryStepRow`, `StoryToolCallRow`, `StorylineTables`, and the +three table-name constants. The root-level `storyline_schema` module is removed, +and internal Storyline storage code imports the model through its local module. + ## Verification - The CSS/source class-token difference contains no CSS-only `.pc2-*` class. @@ -70,5 +123,10 @@ the feature policy. clean for all four supported pChronicle feature combinations. - Workspace strict Clippy remains green with `persisting-dlcapt` excluded. - pChronicle library, pChronicle CLI, and pChronicle Web tests remain green. +- Gateway tests that consume the root-level AgenticMD API remain green. +- No AgenticMD implementation file remains under `formats`, `mapping`, + `layout`, `store`, `convert`, or `projection`; only the private + `agenticmd/` subtree owns that domain. +- `storyline_schema.rs` is gone and Storyline model/Arrow code lives together + under `store/storyline/` while root-level item exports remain available. - Existing user-owned untracked review and RFC files remain untouched. - From 66e242fbc9395140b501d60e81101f2aa79f52ef Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:08:04 +0800 Subject: [PATCH 07/65] docs: translate pchronicle cleanup design --- ...17-pchronicle-post-judge-cleanup-design.md | 179 ++++++++---------- 1 file changed, 84 insertions(+), 95 deletions(-) diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md index 92016855..af4d9732 100644 --- a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md +++ b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md @@ -1,76 +1,72 @@ -# pChronicle Post-Judge Cleanup Design +# pChronicle Judge 删除后的清理设计 -## Goal +## 目标 -Finish the cleanup selected after removing pChronicle's judge subsystem: -remove orphaned Web CSS, replace the remaining production `unreachable!` calls -with explicit errors, verify supported non-search feature combinations, and -co-locate the fragmented AgenticMD and Storyline row-model implementations. +完成 pChronicle 删除 Judge 子系统后的收尾清理：删除 Web 中失去消费者的 +CSS，使用显式错误替换剩余的生产代码 `unreachable!`，验证所有受支持的非 +Search feature 组合，并将分散的 AgenticMD 实现与 Storyline 行模型分别收拢到 +清晰的领域目录中。 -## Scope +## 范围 -This change includes only: +本次变更仅包括： -- pChronicle Web CSS class selectors with no static consumer anywhere under - `pchronicle-web/src`; -- the four production `unreachable!` sites currently reported by - `clippy::unreachable` in default-feature pChronicle builds; -- pChronicle feature checks for core-only, local Lance, default S3, and OSS - builds; -- the local `just` lint entry point and the matching GitHub Actions lint step; -- the AgenticMD codecs, mapping, layout, file operations, conversion, and - projection code currently spread across six crate areas; -- the Storyline logical row model currently separated from its Arrow codecs. +- 删除 `pchronicle-web/src` 中没有任何静态消费者的 pChronicle Web CSS class + selector； +- 处理默认 feature 构建中由 `clippy::unreachable` 报告的 4 处生产代码 + `unreachable!`； +- 检查 pChronicle 的纯核心、本地 Lance、默认 S3 和 OSS 构建组合； +- 统一本地 `just` lint 入口与对应的 GitHub Actions lint 步骤； +- 收拢当前分散在六个 crate 区域中的 AgenticMD codec、映射、路径、文件操作、 + 转换和投影代码； +- 将 Storyline 逻辑行模型与其 Arrow codec 移到同一个目录。 -Search, TTAS, queues and samplers, and `persisting-dlcapt` remain out of scope. -The source-only `pc2-token-composition` class is also out of scope because it is -not an orphaned CSS selector and removing or styling it would be a UI decision. +Search、TTAS、Queue 与 Sampler，以及 `persisting-dlcapt` 均不在本次范围内。 +只存在于源码中的 `pc2-token-composition` class 也不在范围内，因为它不是孤儿 +CSS selector；删除它或为它补充样式都属于新的 UI 决策。 -## Web CSS Cleanup +## Web CSS 清理 -The cleanup is conservative and token-based. A `.pc2-*` selector is removable -only when its class token does not occur in any Rust source under -`pchronicle-web/src`. This removes the four definite judge remnants -(`pc2-verdict-grid`, `pc2-verdict`, `pc2-rubric-list`, and `pc2-score-track`) -and the other selectors left behind by superseded Explorer layouts. +清理采用保守的 class token 比对。只有当一个 `.pc2-*` selector 的 class token +在 `pchronicle-web/src` 的全部 Rust 源码中均不存在时，才允许删除它。该规则会 +删除 4 个确定的 Judge 残留：`pc2-verdict-grid`、`pc2-verdict`、 +`pc2-rubric-list` 和 `pc2-score-track`，并删除旧版 Explorer 布局遗留的其他无 +消费者 selector。 -No retained selector is renamed, no layout value is changed, and no source -markup is changed. Verification repeats the cross-file class-token comparison -and runs the Web test suite and build checks. +本次不会重命名保留的 selector，不会修改任何布局参数，也不会修改源码 markup。 +验证阶段会重新执行跨文件 class token 比对，并运行 Web 测试与构建检查。 -## Explicit Error Semantics +## 显式错误语义 -Each remaining production `unreachable!` becomes an ordinary error at the -same abstraction boundary: +剩余的每处生产代码 `unreachable!` 都在原抽象边界转为普通错误： -- projection sync reports incompatible non-canonical lineage; -- local manifest construction rejects an unsupported query format; -- Catalog projection binding reports a source-kind inconsistency; -- OpenAI corpus recovery rejects an unknown retained document kind. +- Projection 同步在遇到非 canonical lineage 时报告不兼容错误； +- 本地 manifest 构建拒绝不受支持的查询格式； +- Catalog projection 绑定报告 source kind 内部不一致； +- OpenAI corpus 恢复拒绝未知的已保留文档类型。 -The successful path remains unchanged. After conversion, the pChronicle panic -lint also denies `clippy::unreachable`, alongside `unwrap_used` and -`expect_used`. +成功路径的行为保持不变。转换完成后，pChronicle panic 门禁在现有 +`clippy::unwrap_used` 和 `clippy::expect_used` 之外继续 deny +`clippy::unreachable`。 -## Feature Matrix +## Feature 矩阵 -The local lint entry point and CI verify these non-search configurations: +本地 lint 入口与 CI 验证以下非 Search 配置： -1. `--no-default-features` for the lightweight format/event surface; -2. `--no-default-features --features lance-store` for local storage; -3. default features for the S3-compatible product build; -4. `--no-default-features --features oss-store` for the OSS backend. +1. 使用 `--no-default-features` 验证轻量格式与事件表面； +2. 使用 `--no-default-features --features lance-store` 验证本地存储； +3. 使用默认 feature 验证兼容 S3 的产品构建； +4. 使用 `--no-default-features --features oss-store` 验证 OSS 后端。 -Each variant builds the library with warnings denied and the production panic -lints enabled. The workspace-wide strict Clippy command remains the first gate. -CI invokes the same `just lint-rust` recipe used locally, avoiding two copies of -the feature policy. +每种配置都以 warnings deny 和生产 panic lint 开启的方式构建 library。 +workspace 级严格 Clippy 仍是第一道门禁。CI 调用与本地相同的 +`just lint-rust` recipe，避免维护两份 feature 策略。 -## AgenticMD Domain Consolidation +## AgenticMD 领域收拢 -AgenticMD currently spans roughly 2,465 lines across `formats`, `mapping`, -`layout`, `store`, `convert`, and `projection`. The implementation moves into a -single private root subtree with responsibility-oriented files: +AgenticMD 当前约有 2,465 行实现，分散在 `formats`、`mapping`、`layout`、 +`store`、`convert` 和 `projection` 六处。实现统一迁移到 crate 根部的一个私有 +子树，并继续按职责拆分文件： ```text agenticmd/ @@ -89,44 +85,37 @@ agenticmd/ └── projection.rs ``` -`projection.rs` remains gated by `lance-store`. The move changes no wire -format, path rule, mapping rule, filesystem behavior, or projection behavior. -The crate root continues to re-export the existing public AgenticMD functions, -constants, and types, so current Gateway and CLI consumers remain source -compatible. - -Old implementation-oriented module paths such as -`persisting_pchronicle::formats::agenticmd::*` are intentionally not retained -through compatibility wrappers. No workspace consumer uses those paths, the -crate remains pre-1.0, and retaining them would preserve the structure this -change is meant to remove. `formats`, `convert`, `projection`, and `store` may -re-export domain functions at their existing module root where that requires no -wrapper module, but they no longer own AgenticMD implementation files. - -## Storyline Row-Model Consolidation - -The logical Storyline three-table model moves from root-level -`storyline_schema.rs` to `store/storyline/model.rs`. It remains separate from -`store/storyline/rows.rs`: `model.rs` owns normalization, reconstruction, and -logical row types, while `rows.rs` owns Arrow schemas and codecs. Co-location, -not a thousand-line file merge, is the goal. - -The crate root continues to export `split_storyline`, `reconstruct_storyline`, -`StoryRunRow`, `StoryStepRow`, `StoryToolCallRow`, `StorylineTables`, and the -three table-name constants. The root-level `storyline_schema` module is removed, -and internal Storyline storage code imports the model through its local module. - -## Verification - -- The CSS/source class-token difference contains no CSS-only `.pc2-*` class. -- `clippy::unwrap_used`, `clippy::expect_used`, and `clippy::unreachable` are - clean for all four supported pChronicle feature combinations. -- Workspace strict Clippy remains green with `persisting-dlcapt` excluded. -- pChronicle library, pChronicle CLI, and pChronicle Web tests remain green. -- Gateway tests that consume the root-level AgenticMD API remain green. -- No AgenticMD implementation file remains under `formats`, `mapping`, - `layout`, `store`, `convert`, or `projection`; only the private - `agenticmd/` subtree owns that domain. -- `storyline_schema.rs` is gone and Storyline model/Arrow code lives together - under `store/storyline/` while root-level item exports remain available. -- Existing user-owned untracked review and RFC files remain untouched. +`projection.rs` 继续受 `lance-store` feature 门控。本次迁移不改变 wire 格式、 +路径规则、映射规则、文件系统行为或投影行为。crate 根继续 re-export 现有公开的 +AgenticMD 函数、常量与类型，因此当前 Gateway 和 CLI 消费者保持源码兼容。 + +不通过兼容 wrapper 保留 `persisting_pchronicle::formats::agenticmd::*` 等旧的 +实现导向模块路径。workspace 中没有消费者使用这些路径；crate 仍处于 1.0 之前； +保留这些路径也会把本次需要消除的旧结构永久固化。只要无需新增 wrapper 模块， +`formats`、`convert`、`projection` 和 `store` 可以继续在各自模块根 re-export +领域函数，但不再拥有 AgenticMD 实现文件。 + +## Storyline 行模型收拢 + +Storyline 三表逻辑模型从 crate 根部的 `storyline_schema.rs` 移至 +`store/storyline/model.rs`。它与 `store/storyline/rows.rs` 保持为两个文件： +`model.rs` 负责规范化、重建和逻辑行类型，`rows.rs` 负责 Arrow schema 与 codec。 +本次目标是同域代码同目录，而不是合并成一个上千行文件。 + +crate 根继续导出 `split_storyline`、`reconstruct_storyline`、`StoryRunRow`、 +`StoryStepRow`、`StoryToolCallRow`、`StorylineTables` 和三个表名常量。删除根级 +`storyline_schema` 模块，Storyline 存储内部通过同目录模块导入逻辑模型。 + +## 验收与验证 + +- CSS 与源码 class token 的差集不再包含任何仅存在于 CSS 的 `.pc2-*` class； +- `clippy::unwrap_used`、`clippy::expect_used` 和 `clippy::unreachable` 在四种 + pChronicle feature 组合中均通过； +- 排除 `persisting-dlcapt` 的 workspace 严格 Clippy 保持通过； +- pChronicle library、pChronicle CLI 和 pChronicle Web 测试保持通过； +- 使用 AgenticMD 根级 API 的 Gateway 测试保持通过； +- `formats`、`mapping`、`layout`、`store`、`convert` 和 `projection` 下不再保留 + AgenticMD 实现文件，只有私有 `agenticmd/` 子树拥有该领域实现； +- `storyline_schema.rs` 被删除，Storyline 模型与 Arrow 代码共同位于 + `store/storyline/`，同时保留根级 item 导出； +- 现有用户未跟踪的评审稿和 RFC 文件保持不变。 From 5e26e47b936ce9c1870557d045724eb389678d74 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:11:25 +0800 Subject: [PATCH 08/65] docs: plan pchronicle post-judge cleanup --- ...026-08-17-pchronicle-post-judge-cleanup.md | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md diff --git a/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md b/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md new file mode 100644 index 00000000..42ad0173 --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md @@ -0,0 +1,397 @@ +# pChronicle Judge 删除后的清理实施计划 + +> **供 agentic worker 使用：** 必须使用 `superpowers:subagent-driven-development`（推荐）或 `superpowers:executing-plans`，逐项执行本计划。所有步骤使用 checkbox 跟踪。 + +**目标：** 清理 pChronicle Web 孤儿 CSS，消除剩余生产 `unreachable!`，补齐非 Search feature 门禁，并收拢 AgenticMD 与 Storyline 行模型的物理目录边界。 + +**架构：** 所有改动保持现有根级公共 item API 和运行时语义。AgenticMD 实现统一迁入私有 `agenticmd/` 子树；Storyline 逻辑模型迁入 `store/storyline/model.rs` 并与 Arrow codec 并列。CI 只调用本地 `just lint-rust`，由该 recipe 定义完整 feature 矩阵。 + +**技术栈：** Rust 2021、Cargo、Clippy、Dioxus Web、GitHub Actions、Just。 + +## 全局约束 + +- 不进入 Search、TTAS、Queue、Sampler 或 `persisting-dlcapt`。 +- 不修改、删除或提交用户已有的未跟踪文件。 +- 不改变 AgenticMD wire、路径、映射、文件或投影语义。 +- 保留当前 crate 根级 AgenticMD 和 Storyline item 导出。 +- 不为旧的深层实现路径增加兼容 wrapper。 +- 每个结构迁移先运行已有行为测试建立绿色基线，迁移后运行同一测试；不提交只检测私有文件路径的 change-detector 测试。 + +--- + +### 任务 1：删除 Web 孤儿 CSS + +**文件：** +- 修改：`pchronicle-web/assets/analysis.css` +- 修改：`pchronicle-web/assets/path-explorer.css` +- 修改：`pchronicle-web/assets/workbench.css` + +**接口：** +- 输入：`pchronicle-web/src` 中出现的静态 `pc2-*` class token。 +- 输出：CSS 中不存在无源码消费者的 `.pc2-*` selector。 + +- [ ] **步骤 1：运行 class 差集检查，确认 RED** + +运行： + +```bash +comm -23 \ + <(rg --no-filename -o '\.pc2-[a-z0-9-]+' pchronicle-web/assets/*.css | sed 's/^\.//' | sort -u) \ + <(rg --no-filename -o 'pc2-[a-z0-9-]+' pchronicle-web/src | sort -u) +``` + +预期：输出 25 个 CSS-only class，其中包括 `pc2-verdict-grid`、 +`pc2-verdict`、`pc2-rubric-list` 和 `pc2-score-track`。 + +- [ ] **步骤 2：只删除 RED 列表中的 selector 规则** + +从三个 CSS 文件删除所有引用 RED class 的完整规则；组合 selector 中只删除对应 +分支，保留仍有消费者的 selector 和声明。不得修改 Rust markup，也不得删除 +source-only 的 `pc2-token-composition`。 + +- [ ] **步骤 3：运行 class 差集检查，确认 GREEN** + +重复步骤 1 的命令。 + +预期：无输出。 + +- [ ] **步骤 4：验证 Web** + +运行： + +```bash +cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check +cargo test --manifest-path pchronicle-web/Cargo.toml --locked +``` + +预期：格式检查和全部 Web 测试通过。 + +- [ ] **步骤 5：提交** + +```bash +git add pchronicle-web/assets/analysis.css \ + pchronicle-web/assets/path-explorer.css \ + pchronicle-web/assets/workbench.css +git commit -m "style: remove orphaned pchronicle web selectors" +``` + +### 任务 2：消除生产 `unreachable!` + +**文件：** +- 修改：`crates/persisting-pchronicle/src/projection/storyline.rs` +- 修改：`crates/persisting-pchronicle/src/store/local_query_manifest.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/discovery.rs` +- 修改：`crates/persisting-pchronicle/src/formats/openai_corpus.rs` + +**接口：** +- 输入：现有四处内部不变量。 +- 输出：同一函数的普通 `Result` 错误，不再触发进程 panic。 + +- [ ] **步骤 1：运行严格 lint，确认 RED** + +运行： + +```bash +cargo clippy -p persisting-pchronicle --lib --locked -- \ + -D warnings \ + -D clippy::unwrap_used \ + -D clippy::expect_used \ + -D clippy::unreachable +``` + +预期：仅因四处 `unreachable!` 失败。 + +- [ ] **步骤 2：将 Projection lineage 分支改为显式错误** + +在 `sync_storyline_projection` 中使用返回错误的 `let ... else`： + +```rust +let ProjectionSourceSnapshot::CanonicalEvents { + fact_version: previous_fact_version, + fact_rows: previous_fact_rows, + .. +} = &previous.source +else { + anyhow::bail!("projection source is not canonical events; use `project rebuild`"); +}; +``` + +- [ ] **步骤 3：将其他三处不变量改为显式错误** + +本地 manifest 对不受支持格式返回包含 format 与输入路径的 `anyhow` 错误；Catalog +发现对非 Events source 返回内部一致性错误；OpenAI corpus 恢复对未知 `group.kind` +返回包含 kind 与相对路径的 `Error::Other`。 + +- [ ] **步骤 4：运行严格 lint，确认 GREEN** + +重复步骤 1 命令。 + +预期：通过。 + +- [ ] **步骤 5：运行行为测试并提交** + +运行： + +```bash +cargo test -p persisting-pchronicle --lib --locked +``` + +提交： + +```bash +git add crates/persisting-pchronicle/src/projection/storyline.rs \ + crates/persisting-pchronicle/src/store/local_query_manifest.rs \ + crates/persisting-pchronicle/src/store/catalog/discovery.rs \ + crates/persisting-pchronicle/src/formats/openai_corpus.rs +git commit -m "refactor: replace pchronicle unreachable branches" +``` + +### 任务 3：收拢 Storyline 行模型 + +**文件：** +- 移动：`crates/persisting-pchronicle/src/storyline_schema.rs` → `crates/persisting-pchronicle/src/store/storyline/model.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/mod.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/rows.rs` +- 修改：`crates/persisting-pchronicle/src/store/mod.rs` +- 修改：`crates/persisting-pchronicle/src/lib.rs` + +**接口：** +- 保留：crate 根的 `split_storyline`、`reconstruct_storyline`、四个行/表类型和三个表名常量。 +- 删除：公开模块路径 `persisting_pchronicle::storyline_schema`。 + +- [ ] **步骤 1：运行现有 Storyline 模型测试，建立基线** + +运行： + +```bash +cargo test -p persisting-pchronicle storyline_schema::tests --locked +``` + +预期：全部通过。 + +- [ ] **步骤 2：移动逻辑模型并改为同目录引用** + +将文件移动为 `store/storyline/model.rs`；在 `store/storyline/mod.rs` 声明 +`mod model;`，并从本地 `model` 导入类型与函数。`rows.rs` 使用 +`super::model::{StoryRunRow, StoryStepRow, StoryToolCallRow}`。 + +- [ ] **步骤 3：保持根级 item 导出并删除根模块** + +由 `store/storyline/mod.rs` re-export： + +```rust +pub use model::{ + reconstruct_storyline, split_storyline, StoryRunRow, StoryStepRow, + StoryToolCallRow, StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, + STORY_TOOL_CALLS_TABLE, +}; +``` + +`store/mod.rs` 将这些 item 向上 re-export；`lib.rs` 从 `store` 导出它们并删除 +`pub mod storyline_schema` 及对应导出块。 + +- [ ] **步骤 4：运行迁移后的测试与编译** + +运行： + +```bash +cargo test -p persisting-pchronicle --lib --locked +cargo check -p persisting-pchronicle-cli --tests --locked +``` + +预期：全部通过，且 `storyline_schema.rs` 不存在。 + +- [ ] **步骤 5：提交** + +```bash +git add -u crates/persisting-pchronicle/src +git add crates/persisting-pchronicle/src/store/storyline/model.rs +git commit -m "refactor: colocate storyline row model" +``` + +### 任务 4：收拢 AgenticMD 领域实现 + +**文件：** +- 新建目录：`crates/persisting-pchronicle/src/agenticmd/` +- 移动：现有 AgenticMD codec、body、frontmatter、validation、mapping、layout、filesystem、conversion 与 projection 实现文件 +- 修改：`formats/mod.rs`、`convert/mod.rs`、`layout/mod.rs`、`projection/mod.rs`、`store/mod.rs`、`lib.rs` +- 删除：根级 `mapping/` 模块与各旧 AgenticMD 实现文件 + +**接口：** +- 保留：当前 crate 根级 AgenticMD types、constants、codec、mapping、path、filesystem、conversion 与 projection item。 +- 删除：`formats::agenticmd*` 等实现导向深层模块路径。 + +- [ ] **步骤 1：运行 AgenticMD 与 Gateway 行为测试，建立基线** + +运行： + +```bash +cargo test -p persisting-pchronicle agenticmd --locked +cargo test -p persisting-gateway --lib projection:: --locked +cargo test -p persisting-gateway --test agenticmd_bridge --locked +cargo test -p persisting-gateway --test agenticmd_golden --locked +cargo test -p persisting-gateway --test markdown_trajectory --locked +``` + +预期：全部通过。 + +- [ ] **步骤 2：创建私有领域模块并移动实现文件** + +按规格创建： + +```text +agenticmd/{codec,body,frontmatter,validate,layout,fs,convert,projection}.rs +agenticmd/mapping/{mod,fields,text}.rs +``` + +`agenticmd/mod.rs` 使用私有子模块，并以 `pub(crate) use` 或 `pub use` 聚合 crate +内部和根门面需要的 item；`projection` 保持 `#[cfg(feature = "lance-store")]`。 + +- [ ] **步骤 3：改写领域内部引用** + +领域文件优先使用 `super` 或 `crate::agenticmd` 引用同域 item；Storyline、EventRecord +和 Store 类型继续从其所属模块导入。删除对旧 `crate::formats::agenticmd*`、 +`crate::mapping` 和 `crate::layout::markdown` 的依赖。 + +- [ ] **步骤 4：重接公共门面** + +`lib.rs` 增加私有 `mod agenticmd;`，删除 `pub mod mapping;`。根级 re-export +继续提供现有公共 item。`formats`、`convert`、`layout`、`projection` 和 `store` +只在其模块根直接 re-export 仍需保留的 item，不创建旧深层模块 wrapper。 + +- [ ] **步骤 5：运行 AgenticMD 与 Gateway 测试，确认行为不变** + +重复步骤 1 的全部测试，并运行： + +```bash +cargo check -p persisting-pchronicle --no-default-features --locked +cargo check -p persisting-pchronicle-cli --tests --locked +``` + +预期：全部通过；旧实现文件和根级 `mapping/` 不存在。 + +- [ ] **步骤 6：提交** + +```bash +git add -u crates/persisting-pchronicle/src +git add crates/persisting-pchronicle/src/agenticmd +git commit -m "refactor: consolidate agenticmd domain" +``` + +### 任务 5：补齐 feature 与 panic 门禁 + +**文件：** +- 修改：`justfile` +- 修改：`.github/workflows/ci.yml` + +**接口：** +- 输入：workspace strict Clippy 与四种 pChronicle 非 Search feature 组合。 +- 输出：本地和 CI 共用的 `just lint-rust` 门禁。 + +- [ ] **步骤 1：确认当前门禁缺少 feature 矩阵** + +运行： + +```bash +just --dry-run lint-rust +``` + +预期：仅包含 workspace Clippy 和默认 feature pChronicle panic lint。 + +- [ ] **步骤 2：扩展 Just recipe** + +使 `lint-rust` 依赖 workspace strict Clippy、默认 panic lint 和 +`clippy-pchronicle-features`。所有 pChronicle library 命令统一使用： + +```text +-D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable +``` + +feature recipe 依次检查： + +```text +--no-default-features +--no-default-features --features lance-store +--no-default-features --features oss-store +``` + +默认 S3 配置由现有 `clippy-pchronicle-panics` 覆盖。 + +- [ ] **步骤 3：让 CI 使用唯一入口** + +将 `.github/workflows/ci.yml` 的 Rust clippy step 改为： + +```yaml +- name: Rust clippy + run: just lint-rust +``` + +- [ ] **步骤 4：运行门禁** + +运行： + +```bash +just --dry-run lint-rust +just lint-rust +``` + +预期：workspace 与四种 pChronicle feature 配置全部通过。 + +- [ ] **步骤 5：提交** + +```bash +git add justfile .github/workflows/ci.yml +git commit -m "ci: check pchronicle feature matrix" +``` + +### 任务 6：最终验证与审查 + +**文件：** 不新增生产改动。 + +- [ ] **步骤 1：格式与残留扫描** + +运行： + +```bash +cargo fmt --all -- --check +cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check +rg -n '\b(unreachable!|panic!|todo!|unimplemented!)' \ + crates/persisting-pchronicle/src --glob '*.rs' --glob '!search/**' +``` + +预期：生产代码不再包含 `unreachable!`；测试中的 `panic!` 可保留。 + +- [ ] **步骤 2：核心与 CLI 测试** + +运行： + +```bash +cargo test -p persisting-pchronicle --lib --locked +cargo test -p persisting-pchronicle-cli --lib --tests --locked +cargo test --manifest-path pchronicle-web/Cargo.toml --locked +``` + +CLI loopback 测试如受 sandbox 限制，使用已授权的沙箱外 `cargo test` 重跑完整套件。 + +- [ ] **步骤 3：Gateway 回归** + +运行： + +```bash +cargo test -p persisting-gateway --lib projection:: --locked +cargo test -p persisting-gateway --test agenticmd_bridge --locked +cargo test -p persisting-gateway --test agenticmd_golden --locked +cargo test -p persisting-gateway --test markdown_trajectory --locked +``` + +- [ ] **步骤 4：最终门禁和 diff 审查** + +运行： + +```bash +just lint-rust +git diff --check +git status --short +``` + +确认只有用户原有未跟踪文件，所有计划内改动均已提交。 From dfcaea7d13a9c65cc4ed8010532695ba41b9deba Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:13:05 +0800 Subject: [PATCH 09/65] style: remove orphaned pchronicle web selectors --- pchronicle-web/assets/analysis.css | 2 +- pchronicle-web/assets/path-explorer.css | 2 +- pchronicle-web/assets/workbench.css | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pchronicle-web/assets/analysis.css b/pchronicle-web/assets/analysis.css index 0931422d..c554d7d0 100644 --- a/pchronicle-web/assets/analysis.css +++ b/pchronicle-web/assets/analysis.css @@ -1 +1 @@ -.pc2-detail-tabs{display:flex;align-items:center;gap:3px;margin:0 20px 10px;border-bottom:1px solid #dfe3e8}.pc2-detail-tabs button{position:relative;padding:8px 12px;border:0;background:transparent;color:#667085;font-size:10px;font-weight:700;cursor:pointer}.pc2-detail-tabs button:after{position:absolute;right:7px;bottom:-1px;left:7px;height:2px;border-radius:2px;background:transparent;content:""}.pc2-detail-tabs button.active{color:#1d4ed8}.pc2-detail-tabs button.active:after{background:#2563eb}.pc2-detail-tabs button:focus-visible,.pc2-analysis-tabs button:focus-visible,.pc2-turn-bars button:focus-visible{outline:2px solid #60a5fa;outline-offset:2px}.pc2-detail-tabs>span{margin-left:auto;color:#98a2b3;font-size:9px}.pc2-analysis-workspace{min-height:0;display:flex;flex:1;flex-direction:column;margin:0 20px 18px;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden}.pc2-analysis-tabs{display:flex;align-items:center;gap:4px;padding:8px 11px;border-bottom:1px solid #e8ebef;background:#f8fafc}.pc2-analysis-tabs button{padding:6px 10px;border:1px solid transparent;border-radius:6px;background:transparent;color:#667085;font-size:9px;font-weight:700;cursor:pointer}.pc2-analysis-tabs button:hover{background:#fff;color:#344054}.pc2-analysis-tabs button.active{border-color:#bfdbfe;background:#eff6ff;color:#1d4ed8;box-shadow:0 1px 2px #1018280a}.pc2-analysis-scroll{min-height:0;flex:1;padding:12px;overflow:auto;scrollbar-gutter:stable}.pc2-analysis-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:11px}.pc2-analysis-card{min-width:0;padding:13px;border:1px solid #e4e7ec;border-radius:9px;background:#fff;box-shadow:0 1px 2px #10182808}.pc2-analysis-card.wide{grid-column:1/-1}.pc2-analysis-card>header{min-height:35px;display:flex;align-items:flex-start;justify-content:space-between;gap:12px;margin-bottom:11px}.pc2-analysis-card h3{margin:0;color:#1d2939;font-size:11px}.pc2-analysis-card header p{margin:3px 0 0;color:#98a2b3;font-size:8px;line-height:1.4}.pc2-analysis-card header>span,.pc2-analysis-card header>strong{color:#667085;font:600 9px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-analysis-empty{min-height:90px;display:flex;align-items:center;justify-content:center;color:#98a2b3;font-size:9px;text-align:center}.pc2-dimension-list{display:flex;flex-direction:column;gap:9px}.pc2-dimension-row>div{display:flex;align-items:center;justify-content:space-between;gap:8px}.pc2-dimension-row>div span{overflow:hidden;color:#475467;font-size:9px;font-weight:600;text-overflow:ellipsis;white-space:nowrap}.pc2-dimension-row code{color:#344054;font-size:9px}.pc2-dimension-track,.pc2-coverage-track,.pc2-score-track,.pc2-mini-track{display:block;height:6px;margin-top:4px;border-radius:999px;background:#eef1f5;overflow:hidden}.pc2-dimension-track i,.pc2-coverage-track i,.pc2-score-track i,.pc2-mini-track i{display:block;height:100%;border-radius:inherit;background:#60a5fa}.pc2-dimension-list.violet .pc2-dimension-track i{background:#a78bfa}.pc2-dimension-list.green .pc2-dimension-track i{background:#34d399}.pc2-dimension-list.amber .pc2-dimension-track i{background:#f59e0b}.pc2-dimension-list.red .pc2-dimension-track i{background:#ef4444}.pc2-dimension-row small{display:block;margin-top:3px;color:#98a2b3;font-size:7px}.pc2-coverage-list{display:flex;flex-direction:column;gap:11px}.pc2-coverage-list>div>div{display:flex;justify-content:space-between;color:#667085;font-size:9px}.pc2-coverage-list code{color:#344054;font-size:8px}.pc2-coverage-track{height:7px;margin-top:5px}.pc2-coverage-track i{background:linear-gradient(90deg,#2563eb,#60a5fa)}.pc2-run-span>code{display:block;padding:10px;border:1px solid #e4e7ec;border-radius:7px;background:#f8fafc;color:#344054;font-size:9px;white-space:normal;word-break:break-word}.pc2-run-span-meta{display:flex;flex-wrap:wrap;gap:8px;margin-top:10px}.pc2-run-span-meta span{padding:5px 7px;border-radius:6px;background:#f2f4f7;color:#667085;font-size:8px}.pc2-run-span-meta strong{color:#344054;font-size:9px}.pc2-histogram{height:155px;display:flex;align-items:stretch;gap:7px;padding-top:8px}.pc2-histogram-column{min-width:0;display:grid;grid-template-rows:16px 1fr 22px;flex:1;text-align:center}.pc2-histogram-column>span{color:#667085;font:600 8px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-histogram-column>div{display:flex;align-items:flex-end;border-bottom:1px solid #d0d5dd;background:linear-gradient(180deg,transparent,#f8fafc)}.pc2-histogram-column i{width:70%;min-height:2px;margin:0 auto;border-radius:4px 4px 0 0;background:linear-gradient(180deg,#60a5fa,#2563eb)}.pc2-histogram-column small{padding-top:5px;color:#98a2b3;font-size:7px;white-space:nowrap}.pc2-percentiles{display:flex;flex-direction:column;gap:12px}.pc2-percentile>div{display:flex;align-items:center;justify-content:space-between;color:#667085;font-size:9px}.pc2-percentile code{color:#344054;font-size:9px}.pc2-percentile>span{display:block;height:8px;margin-top:4px;border-radius:999px;background:#eef1f5;overflow:hidden}.pc2-percentile i{display:block;height:100%;border-radius:inherit;background:linear-gradient(90deg,#93c5fd,#2563eb)}.pc2-percentile-coverage{padding-top:8px;border-top:1px solid #eef0f3;color:#98a2b3;font-size:8px}.pc2-turn-chart{min-height:190px}.pc2-turn-bars{height:180px;display:flex;align-items:flex-end;gap:2px;padding:8px 4px 0;border-bottom:1px solid #d0d5dd;background:repeating-linear-gradient(to top,#f8fafc 0,#f8fafc 1px,transparent 1px,transparent 45px)}.pc2-turn-bars button{min-width:2px;display:flex;align-items:flex-end;align-self:stretch;flex:1;padding:0;border:0;background:transparent;cursor:pointer}.pc2-turn-bars button:hover{background:#eff6ff}.pc2-turn-bars i{width:100%;min-height:2px;border-radius:3px 3px 0 0;background:#60a5fa}.pc2-turn-bars i.agent{background:#34d399}.pc2-turn-bars i.system{background:#fbbf24}.pc2-turn-bars i.user{background:#60a5fa}.pc2-chart-note{margin:6px 0 0;color:#98a2b3;font-size:7px;text-align:right}.pc2-ranked-list{display:flex;flex-direction:column}.pc2-ranked-list>button{display:grid;grid-template-columns:25px minmax(0,1fr) 75px;gap:8px;align-items:center;padding:8px;border:0;border-top:1px solid #eef0f3;background:#fff;text-align:left;cursor:pointer}.pc2-ranked-list>button:first-child{border-top:0}.pc2-ranked-list>button:hover{background:#f7faff}.pc2-rank{width:20px;height:20px;display:grid;place-items:center;border-radius:5px;background:#eff6ff;color:#2563eb;font:700 8px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-ranked-copy{min-width:0;display:flex;flex-direction:column;gap:2px}.pc2-ranked-copy strong,.pc2-ranked-copy small{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.pc2-ranked-copy strong{color:#344054;font-size:9px}.pc2-ranked-copy small{color:#98a2b3;font-size:8px}.pc2-ranked-list code{color:#475467;font-size:9px;text-align:right}.pc2-token-track{height:14px;display:flex;border-radius:999px;background:#eef1f5;overflow:hidden}.pc2-token-track i{height:100%}.pc2-token-track .prompt,.pc2-token-legend .prompt{background:#2563eb}.pc2-token-track .completion,.pc2-token-legend .completion{background:#a78bfa}.pc2-token-legend{display:flex;gap:14px;margin-top:8px;color:#667085;font-size:8px}.pc2-token-legend span{display:flex;align-items:center;gap:5px}.pc2-token-legend i{width:7px;height:7px;border-radius:2px}.pc2-tool-table{min-width:720px}.pc2-tool-head,.pc2-tool-row{display:grid;grid-template-columns:minmax(180px,2fr) repeat(5,minmax(70px,1fr));gap:10px;align-items:center}.pc2-tool-head{padding:7px 8px;border-bottom:1px solid #dfe3e8;color:#98a2b3;font-size:7px;font-weight:700;text-transform:uppercase;letter-spacing:.05em}.pc2-tool-row{min-height:48px;padding:7px 8px;border-bottom:1px solid #eef0f3;color:#475467;font-size:9px}.pc2-tool-row:last-child{border-bottom:0}.pc2-tool-row>div:first-child{min-width:0}.pc2-tool-row strong{display:block;overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-tool-row code{font-size:8px}.pc2-mini-track{width:110px;height:4px}.pc2-tool-errors{width:max-content;padding:2px 6px;border-radius:999px;background:#f2f4f7;color:#667085;font-size:8px}.pc2-tool-errors.active{background:#fff1f0;color:#b42318}.pc2-verdict-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:8px}.pc2-verdict{display:flex;flex-direction:column;align-items:center;padding:13px 8px;border:1px solid #e4e7ec;border-radius:8px;background:#f8fafc}.pc2-verdict>span{width:7px;height:7px;border-radius:50%;background:#98a2b3}.pc2-verdict strong{margin-top:5px;color:#344054;font:600 20px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-verdict small{color:#667085;font-size:8px}.pc2-verdict.pass>span{background:#22c55e}.pc2-verdict.partial>span{background:#f59e0b}.pc2-verdict.fail>span{background:#ef4444}.pc2-rubric-list{display:flex;flex-direction:column;gap:10px}.pc2-rubric-list>div{display:grid;grid-template-columns:minmax(140px,1fr) minmax(180px,2fr) 50px;gap:12px;align-items:center}.pc2-rubric-list>div>div{min-width:0;display:flex;flex-direction:column;gap:2px}.pc2-rubric-list strong{overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-rubric-list span{color:#98a2b3;font-size:7px}.pc2-score-track{height:8px;margin:0}.pc2-score-track i{background:linear-gradient(90deg,#f59e0b,#22c55e)}.pc2-rubric-list code{color:#344054;font-size:9px;text-align:right}@media(max-width:1150px){.pc2-analysis-grid{grid-template-columns:1fr}.pc2-analysis-card.wide{grid-column:auto}.pc2-detail-tabs>span{display:none}}@media(max-width:850px){.pc2-detail-tabs{margin-right:12px;margin-left:12px}.pc2-analysis-workspace{margin-right:12px;margin-left:12px}.pc2-analysis-tabs{overflow-x:auto}.pc2-analysis-tabs button{white-space:nowrap}.pc2-analysis-scroll{padding:8px}.pc2-tool-table{overflow-x:auto}.pc2-rubric-list>div{grid-template-columns:1fr 1.5fr 40px}} +.pc2-detail-tabs{display:flex;align-items:center;gap:3px;margin:0 20px 10px;border-bottom:1px solid #dfe3e8}.pc2-detail-tabs button{position:relative;padding:8px 12px;border:0;background:transparent;color:#667085;font-size:10px;font-weight:700;cursor:pointer}.pc2-detail-tabs button:after{position:absolute;right:7px;bottom:-1px;left:7px;height:2px;border-radius:2px;background:transparent;content:""}.pc2-detail-tabs button.active{color:#1d4ed8}.pc2-detail-tabs button.active:after{background:#2563eb}.pc2-detail-tabs button:focus-visible,.pc2-analysis-tabs button:focus-visible,.pc2-turn-bars button:focus-visible{outline:2px solid #60a5fa;outline-offset:2px}.pc2-detail-tabs>span{margin-left:auto;color:#98a2b3;font-size:9px}.pc2-analysis-workspace{min-height:0;display:flex;flex:1;flex-direction:column;margin:0 20px 18px;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden}.pc2-analysis-tabs{display:flex;align-items:center;gap:4px;padding:8px 11px;border-bottom:1px solid #e8ebef;background:#f8fafc}.pc2-analysis-tabs button{padding:6px 10px;border:1px solid transparent;border-radius:6px;background:transparent;color:#667085;font-size:9px;font-weight:700;cursor:pointer}.pc2-analysis-tabs button:hover{background:#fff;color:#344054}.pc2-analysis-tabs button.active{border-color:#bfdbfe;background:#eff6ff;color:#1d4ed8;box-shadow:0 1px 2px #1018280a}.pc2-analysis-scroll{min-height:0;flex:1;padding:12px;overflow:auto;scrollbar-gutter:stable}.pc2-analysis-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:11px}.pc2-analysis-card{min-width:0;padding:13px;border:1px solid #e4e7ec;border-radius:9px;background:#fff;box-shadow:0 1px 2px #10182808}.pc2-analysis-card.wide{grid-column:1/-1}.pc2-analysis-card>header{min-height:35px;display:flex;align-items:flex-start;justify-content:space-between;gap:12px;margin-bottom:11px}.pc2-analysis-card h3{margin:0;color:#1d2939;font-size:11px}.pc2-analysis-card header p{margin:3px 0 0;color:#98a2b3;font-size:8px;line-height:1.4}.pc2-analysis-card header>span,.pc2-analysis-card header>strong{color:#667085;font:600 9px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-analysis-empty{min-height:90px;display:flex;align-items:center;justify-content:center;color:#98a2b3;font-size:9px;text-align:center}.pc2-dimension-list{display:flex;flex-direction:column;gap:9px}.pc2-dimension-row>div{display:flex;align-items:center;justify-content:space-between;gap:8px}.pc2-dimension-row>div span{overflow:hidden;color:#475467;font-size:9px;font-weight:600;text-overflow:ellipsis;white-space:nowrap}.pc2-dimension-row code{color:#344054;font-size:9px}.pc2-dimension-track,.pc2-coverage-track,.pc2-mini-track{display:block;height:6px;margin-top:4px;border-radius:999px;background:#eef1f5;overflow:hidden}.pc2-dimension-track i,.pc2-coverage-track i,.pc2-mini-track i{display:block;height:100%;border-radius:inherit;background:#60a5fa}.pc2-dimension-list.violet .pc2-dimension-track i{background:#a78bfa}.pc2-dimension-list.green .pc2-dimension-track i{background:#34d399}.pc2-dimension-list.amber .pc2-dimension-track i{background:#f59e0b}.pc2-dimension-list.red .pc2-dimension-track i{background:#ef4444}.pc2-dimension-row small{display:block;margin-top:3px;color:#98a2b3;font-size:7px}.pc2-coverage-list{display:flex;flex-direction:column;gap:11px}.pc2-coverage-list>div>div{display:flex;justify-content:space-between;color:#667085;font-size:9px}.pc2-coverage-list code{color:#344054;font-size:8px}.pc2-coverage-track{height:7px;margin-top:5px}.pc2-coverage-track i{background:linear-gradient(90deg,#2563eb,#60a5fa)}.pc2-run-span>code{display:block;padding:10px;border:1px solid #e4e7ec;border-radius:7px;background:#f8fafc;color:#344054;font-size:9px;white-space:normal;word-break:break-word}.pc2-run-span-meta{display:flex;flex-wrap:wrap;gap:8px;margin-top:10px}.pc2-run-span-meta span{padding:5px 7px;border-radius:6px;background:#f2f4f7;color:#667085;font-size:8px}.pc2-run-span-meta strong{color:#344054;font-size:9px}.pc2-histogram{height:155px;display:flex;align-items:stretch;gap:7px;padding-top:8px}.pc2-histogram-column{min-width:0;display:grid;grid-template-rows:16px 1fr 22px;flex:1;text-align:center}.pc2-histogram-column>span{color:#667085;font:600 8px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-histogram-column>div{display:flex;align-items:flex-end;border-bottom:1px solid #d0d5dd;background:linear-gradient(180deg,transparent,#f8fafc)}.pc2-histogram-column i{width:70%;min-height:2px;margin:0 auto;border-radius:4px 4px 0 0;background:linear-gradient(180deg,#60a5fa,#2563eb)}.pc2-histogram-column small{padding-top:5px;color:#98a2b3;font-size:7px;white-space:nowrap}.pc2-percentiles{display:flex;flex-direction:column;gap:12px}.pc2-percentile>div{display:flex;align-items:center;justify-content:space-between;color:#667085;font-size:9px}.pc2-percentile code{color:#344054;font-size:9px}.pc2-percentile>span{display:block;height:8px;margin-top:4px;border-radius:999px;background:#eef1f5;overflow:hidden}.pc2-percentile i{display:block;height:100%;border-radius:inherit;background:linear-gradient(90deg,#93c5fd,#2563eb)}.pc2-percentile-coverage{padding-top:8px;border-top:1px solid #eef0f3;color:#98a2b3;font-size:8px}.pc2-turn-chart{min-height:190px}.pc2-turn-bars{height:180px;display:flex;align-items:flex-end;gap:2px;padding:8px 4px 0;border-bottom:1px solid #d0d5dd;background:repeating-linear-gradient(to top,#f8fafc 0,#f8fafc 1px,transparent 1px,transparent 45px)}.pc2-turn-bars button{min-width:2px;display:flex;align-items:flex-end;align-self:stretch;flex:1;padding:0;border:0;background:transparent;cursor:pointer}.pc2-turn-bars button:hover{background:#eff6ff}.pc2-turn-bars i{width:100%;min-height:2px;border-radius:3px 3px 0 0;background:#60a5fa}.pc2-turn-bars i.agent{background:#34d399}.pc2-turn-bars i.system{background:#fbbf24}.pc2-turn-bars i.user{background:#60a5fa}.pc2-chart-note{margin:6px 0 0;color:#98a2b3;font-size:7px;text-align:right}.pc2-ranked-list{display:flex;flex-direction:column}.pc2-ranked-list>button{display:grid;grid-template-columns:25px minmax(0,1fr) 75px;gap:8px;align-items:center;padding:8px;border:0;border-top:1px solid #eef0f3;background:#fff;text-align:left;cursor:pointer}.pc2-ranked-list>button:first-child{border-top:0}.pc2-ranked-list>button:hover{background:#f7faff}.pc2-rank{width:20px;height:20px;display:grid;place-items:center;border-radius:5px;background:#eff6ff;color:#2563eb;font:700 8px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-ranked-copy{min-width:0;display:flex;flex-direction:column;gap:2px}.pc2-ranked-copy strong,.pc2-ranked-copy small{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.pc2-ranked-copy strong{color:#344054;font-size:9px}.pc2-ranked-copy small{color:#98a2b3;font-size:8px}.pc2-ranked-list code{color:#475467;font-size:9px;text-align:right}.pc2-token-track{height:14px;display:flex;border-radius:999px;background:#eef1f5;overflow:hidden}.pc2-token-track i{height:100%}.pc2-token-track .prompt,.pc2-token-legend .prompt{background:#2563eb}.pc2-token-track .completion,.pc2-token-legend .completion{background:#a78bfa}.pc2-token-legend{display:flex;gap:14px;margin-top:8px;color:#667085;font-size:8px}.pc2-token-legend span{display:flex;align-items:center;gap:5px}.pc2-token-legend i{width:7px;height:7px;border-radius:2px}.pc2-tool-table{min-width:720px}.pc2-tool-head,.pc2-tool-row{display:grid;grid-template-columns:minmax(180px,2fr) repeat(5,minmax(70px,1fr));gap:10px;align-items:center}.pc2-tool-head{padding:7px 8px;border-bottom:1px solid #dfe3e8;color:#98a2b3;font-size:7px;font-weight:700;text-transform:uppercase;letter-spacing:.05em}.pc2-tool-row{min-height:48px;padding:7px 8px;border-bottom:1px solid #eef0f3;color:#475467;font-size:9px}.pc2-tool-row:last-child{border-bottom:0}.pc2-tool-row>div:first-child{min-width:0}.pc2-tool-row strong{display:block;overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-tool-row code{font-size:8px}.pc2-mini-track{width:110px;height:4px}.pc2-tool-errors{width:max-content;padding:2px 6px;border-radius:999px;background:#f2f4f7;color:#667085;font-size:8px}.pc2-tool-errors.active{background:#fff1f0;color:#b42318}@media(max-width:1150px){.pc2-analysis-grid{grid-template-columns:1fr}.pc2-analysis-card.wide{grid-column:auto}.pc2-detail-tabs>span{display:none}}@media(max-width:850px){.pc2-detail-tabs{margin-right:12px;margin-left:12px}.pc2-analysis-workspace{margin-right:12px;margin-left:12px}.pc2-analysis-tabs{overflow-x:auto}.pc2-analysis-tabs button{white-space:nowrap}.pc2-analysis-scroll{padding:8px}.pc2-tool-table{overflow-x:auto}} diff --git a/pchronicle-web/assets/path-explorer.css b/pchronicle-web/assets/path-explorer.css index 76650d0b..ddd6bef9 100644 --- a/pchronicle-web/assets/path-explorer.css +++ b/pchronicle-web/assets/path-explorer.css @@ -1 +1 @@ -.pc2-runs-layout{min-height:0;display:grid;grid-template-columns:270px minmax(0,1fr);gap:12px;flex:1}.pc2-runs-main{min-width:0;min-height:0;display:flex;flex-direction:column}.pc2-runs-main .pc2-filterbar{min-height:36px}.pc2-path-explorer{min-width:0;min-height:0;display:flex;flex-direction:column;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden;box-shadow:0 1px 2px #10182808}.pc2-path-explorer>header{min-height:55px;display:flex;align-items:center;justify-content:space-between;gap:8px;padding:9px 11px;border-bottom:1px solid #eceef1;background:#f8fafc}.pc2-path-explorer>header>div{min-width:0;display:flex;flex-direction:column;gap:2px}.pc2-path-explorer>header strong{color:#1d2939;font-size:11px}.pc2-path-explorer>header div span{color:#98a2b3;font-size:8px}.pc2-path-explorer>header>span{min-width:23px;padding:3px 6px;border:1px solid #d0d5dd;border-radius:999px;background:#fff;color:#667085;font:600 8px ui-monospace,SFMono-Regular,Menlo,monospace;text-align:center}.pc2-path-explorer>footer{padding:8px 10px;border-top:1px solid #eef0f3;background:#fafbfc;color:#98a2b3;font-size:7px;line-height:1.4}.pc2-path-tree{min-height:0;flex:1;padding:7px 6px 14px;overflow:auto;scrollbar-gutter:stable}.pc2-path-all,.pc2-path-row{width:100%;min-height:31px;display:grid;grid-template-columns:20px minmax(0,1fr) 8px 28px;gap:3px;align-items:center;padding:2px 5px;border:0;border-radius:6px;background:transparent;color:#475467;text-align:left}.pc2-path-all{grid-template-columns:20px minmax(0,1fr) 28px;margin-bottom:4px;cursor:pointer}.pc2-path-all:hover,.pc2-path-row:hover{background:#f2f6fc}.pc2-path-all.active,.pc2-path-row.active{background:#eaf2ff;color:#1d4ed8}.pc2-path-row.branch{background:#f7faff}.pc2-path-all strong{overflow:hidden;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-path-all code,.pc2-path-row>code{color:#98a2b3;font-size:8px;text-align:right}.pc2-path-all.active code,.pc2-path-row.active>code{color:#2563eb}.pc2-path-toggle{width:19px;height:23px;padding:0;border:0;border-radius:4px;background:transparent;color:#98a2b3;font-size:13px;cursor:pointer}.pc2-path-toggle:hover{background:#e4eaf2;color:#475467}.pc2-path-toggle.leaf{display:block}.pc2-path-name{min-width:0;height:27px;display:flex;align-items:center;gap:6px;padding:0;border:0;background:transparent;color:inherit;text-align:left;cursor:pointer}.pc2-path-name>span:last-child{overflow:hidden;font-size:9px;font-weight:600;text-overflow:ellipsis;white-space:nowrap}.pc2-path-icon{position:relative;width:13px;height:11px;flex:none;color:#94a3b8}.pc2-path-icon.root{display:grid;place-items:center;font-size:11px}.pc2-path-icon.folder:before{position:absolute;inset:2px 0 0;border:1px solid currentColor;border-radius:2px;background:#f8fafc;content:""}.pc2-path-icon.folder:after{position:absolute;top:0;left:1px;width:6px;height:4px;border:1px solid currentColor;border-bottom:0;border-radius:2px 2px 0 0;background:#f8fafc;content:""}.pc2-path-icon.run:before{position:absolute;inset:0 1px;border:1px solid currentColor;border-radius:2px;background:#fff;content:""}.pc2-path-icon.run:after{position:absolute;top:3px;left:4px;width:5px;height:1px;background:currentColor;box-shadow:0 3px currentColor;content:""}.pc2-path-row.active .pc2-path-icon{color:#2563eb}.pc2-path-health{width:6px;height:6px;border-radius:50%;background:#98a2b3}.pc2-path-health.good{background:#22c55e}.pc2-path-health.live{background:#3b82f6}.pc2-path-health.bad{background:#ef4444}.pc2-path-children{margin-left:10px;padding-left:5px;border-left:1px solid #dfe3e8}.pc2-path-loading,.pc2-path-empty{min-height:120px;display:flex;align-items:center;justify-content:center;gap:6px;padding:16px;color:#98a2b3;font-size:8px;text-align:center}.pc2-path-loading .spinner{width:12px;height:12px;margin:0}.pc2-path-filter{max-width:180px;height:30px!important;overflow:hidden;padding:0 8px;border-color:#bfdbfe!important;background:#eff6ff!important;color:#1d4ed8!important;font-size:8px!important;text-overflow:ellipsis;white-space:nowrap;cursor:pointer}.pc2-path-all:focus-visible,.pc2-path-toggle:focus-visible,.pc2-path-name:focus-visible{outline:2px solid #60a5fa;outline-offset:1px}@media(max-width:1150px){.pc2-runs-layout{grid-template-columns:230px minmax(0,1fr)}.pc2-filter-search{min-width:240px}}@media(max-width:850px){.pc2-runs-layout{display:flex;overflow:auto;flex-direction:column}.pc2-path-explorer{min-height:220px;max-height:260px}.pc2-runs-main{min-height:560px}.pc2-path-explorer>footer{display:none}} +.pc2-runs-layout{min-height:0;display:grid;grid-template-columns:270px minmax(0,1fr);gap:12px;flex:1}.pc2-path-explorer{min-width:0;min-height:0;display:flex;flex-direction:column;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden;box-shadow:0 1px 2px #10182808}.pc2-path-explorer>header{min-height:55px;display:flex;align-items:center;justify-content:space-between;gap:8px;padding:9px 11px;border-bottom:1px solid #eceef1;background:#f8fafc}.pc2-path-explorer>header>div{min-width:0;display:flex;flex-direction:column;gap:2px}.pc2-path-explorer>header strong{color:#1d2939;font-size:11px}.pc2-path-explorer>header div span{color:#98a2b3;font-size:8px}.pc2-path-explorer>header>span{min-width:23px;padding:3px 6px;border:1px solid #d0d5dd;border-radius:999px;background:#fff;color:#667085;font:600 8px ui-monospace,SFMono-Regular,Menlo,monospace;text-align:center}.pc2-path-explorer>footer{padding:8px 10px;border-top:1px solid #eef0f3;background:#fafbfc;color:#98a2b3;font-size:7px;line-height:1.4}.pc2-path-tree{min-height:0;flex:1;padding:7px 6px 14px;overflow:auto;scrollbar-gutter:stable}.pc2-path-all,.pc2-path-row{width:100%;min-height:31px;display:grid;grid-template-columns:20px minmax(0,1fr) 8px 28px;gap:3px;align-items:center;padding:2px 5px;border:0;border-radius:6px;background:transparent;color:#475467;text-align:left}.pc2-path-all{grid-template-columns:20px minmax(0,1fr) 28px;margin-bottom:4px;cursor:pointer}.pc2-path-all:hover,.pc2-path-row:hover{background:#f2f6fc}.pc2-path-all.active,.pc2-path-row.active{background:#eaf2ff;color:#1d4ed8}.pc2-path-row.branch{background:#f7faff}.pc2-path-all strong{overflow:hidden;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-path-all code,.pc2-path-row>code{color:#98a2b3;font-size:8px;text-align:right}.pc2-path-all.active code,.pc2-path-row.active>code{color:#2563eb}.pc2-path-toggle{width:19px;height:23px;padding:0;border:0;border-radius:4px;background:transparent;color:#98a2b3;font-size:13px;cursor:pointer}.pc2-path-toggle:hover{background:#e4eaf2;color:#475467}.pc2-path-toggle.leaf{display:block}.pc2-path-name{min-width:0;height:27px;display:flex;align-items:center;gap:6px;padding:0;border:0;background:transparent;color:inherit;text-align:left;cursor:pointer}.pc2-path-name>span:last-child{overflow:hidden;font-size:9px;font-weight:600;text-overflow:ellipsis;white-space:nowrap}.pc2-path-icon{position:relative;width:13px;height:11px;flex:none;color:#94a3b8}.pc2-path-icon.root{display:grid;place-items:center;font-size:11px}.pc2-path-icon.folder:before{position:absolute;inset:2px 0 0;border:1px solid currentColor;border-radius:2px;background:#f8fafc;content:""}.pc2-path-icon.folder:after{position:absolute;top:0;left:1px;width:6px;height:4px;border:1px solid currentColor;border-bottom:0;border-radius:2px 2px 0 0;background:#f8fafc;content:""}.pc2-path-icon.run:before{position:absolute;inset:0 1px;border:1px solid currentColor;border-radius:2px;background:#fff;content:""}.pc2-path-icon.run:after{position:absolute;top:3px;left:4px;width:5px;height:1px;background:currentColor;box-shadow:0 3px currentColor;content:""}.pc2-path-row.active .pc2-path-icon{color:#2563eb}.pc2-path-health{width:6px;height:6px;border-radius:50%;background:#98a2b3}.pc2-path-health.good{background:#22c55e}.pc2-path-health.live{background:#3b82f6}.pc2-path-health.bad{background:#ef4444}.pc2-path-children{margin-left:10px;padding-left:5px;border-left:1px solid #dfe3e8}.pc2-path-loading,.pc2-path-empty{min-height:120px;display:flex;align-items:center;justify-content:center;gap:6px;padding:16px;color:#98a2b3;font-size:8px;text-align:center}.pc2-path-loading .spinner{width:12px;height:12px;margin:0}.pc2-path-filter{max-width:180px;height:30px!important;overflow:hidden;padding:0 8px;border-color:#bfdbfe!important;background:#eff6ff!important;color:#1d4ed8!important;font-size:8px!important;text-overflow:ellipsis;white-space:nowrap;cursor:pointer}.pc2-path-all:focus-visible,.pc2-path-toggle:focus-visible,.pc2-path-name:focus-visible{outline:2px solid #60a5fa;outline-offset:1px}@media(max-width:1150px){.pc2-runs-layout{grid-template-columns:230px minmax(0,1fr)}.pc2-filter-search{min-width:240px}}@media(max-width:850px){.pc2-runs-layout{display:flex;overflow:auto;flex-direction:column}.pc2-path-explorer{min-height:220px;max-height:260px}.pc2-path-explorer>footer{display:none}} diff --git a/pchronicle-web/assets/workbench.css b/pchronicle-web/assets/workbench.css index 350bac51..b82772d7 100644 --- a/pchronicle-web/assets/workbench.css +++ b/pchronicle-web/assets/workbench.css @@ -1 +1 @@ -.pc2-shell{height:100vh;display:grid;grid-template-columns:56px minmax(0,1fr);background:#f5f7fa;color:#172033}.pc2-main{position:relative;min-width:0;min-height:0;display:flex;flex-direction:column;overflow:hidden}.pc2-global-error{position:absolute;z-index:60;top:12px;left:50%;transform:translateX(-50%);display:flex;align-items:center;gap:10px;max-width:680px;padding:9px 12px;border:1px solid #fecaca;border-radius:9px;background:#fff7f7;color:#991b1b;font-size:11px;box-shadow:0 8px 24px #7f1d1d1a}.pc2-global-error button{margin-left:auto;border:0;background:transparent;color:inherit;font-size:18px;cursor:pointer}.pc2-page,.pc2-detail{min-height:0;display:flex;flex:1;flex-direction:column}.pc2-page{padding:22px 24px 18px;overflow:hidden}.pc2-page-head,.pc2-detail-head{display:flex;align-items:center;justify-content:space-between;gap:18px}.pc2-page-head{margin-bottom:18px}.pc2-page-head h1,.pc2-detail-head h1{margin:2px 0;color:#101828;font-size:22px;line-height:1.2}.pc2-page-head p:not(.eyebrow){margin:5px 0 0;color:#667085;font-size:12px}.pc2-filterbar{display:flex;align-items:center;gap:8px;margin-bottom:12px}.pc2-filterbar select,.pc2-filterbar button,.pc2-filter-search{height:36px;border:1px solid #d7dce3;border-radius:8px;background:#fff;color:#344054;font-size:11px}.pc2-filterbar select{padding:0 30px 0 10px}.pc2-filter-search{min-width:340px;display:flex;align-items:center;gap:7px;padding:0 10px;color:#98a2b3}.pc2-filter-search:focus-within{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb18}.pc2-filter-search input{min-width:0;flex:1;border:0;outline:0;background:transparent;color:#344054}.pc2-sort{padding:0 11px;cursor:pointer}.pc2-result-count{margin-left:auto;color:#667085;font-size:11px}.pc2-table-wrap{min-height:0;flex:1;border:1px solid #dfe3e8;border-radius:11px;background:#fff;overflow:auto;box-shadow:0 1px 2px #10182808}.pc2-run-table{width:100%;border-collapse:collapse;table-layout:fixed}.pc2-run-table th{position:sticky;z-index:2;top:0;padding:10px 12px;border-bottom:1px solid #e4e7ec;background:#f8fafc;color:#667085;text-align:left;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.07em}.pc2-run-table th:first-child{width:26%}.pc2-run-table th:nth-child(2){width:19%}.pc2-run-table th:nth-child(3){width:10%}.pc2-run-table th:nth-child(4),.pc2-run-table th:nth-child(5),.pc2-run-table th:nth-child(6){width:9%}.pc2-run-table td{height:61px;padding:9px 12px;border-bottom:1px solid #eef0f3;color:#475467;font-size:11px;vertical-align:middle}.pc2-run-table tbody tr{cursor:pointer}.pc2-run-table tbody tr:hover,.pc2-run-table tbody tr:focus-visible{outline:0;background:#f7faff;box-shadow:inset 3px 0 #3b82f6}.pc2-run-table td code{display:block;overflow:hidden;color:#667085;font-size:10px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell{min-width:0;display:flex;flex-direction:column;gap:4px}.pc2-session-cell strong{overflow:hidden;color:#1d2939;font-size:11px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell span{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-number{font:600 11px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-table-skeleton td{height:61px;background:linear-gradient(90deg,#fafafa,#f0f3f7,#fafafa);background-size:200% 100%;animation:shimmer 1.4s infinite}.pc2-empty{min-height:180px;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:5px;color:#98a2b3;text-align:center}.pc2-empty strong{color:#475467;font-size:12px}.pc2-empty span{font-size:10px}.pc2-pagination{display:flex;align-items:center;justify-content:flex-end;gap:12px;padding-top:11px;color:#667085;font-size:10px}.pc2-pagination button{padding:6px 9px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;cursor:pointer}.pc2-pagination button:disabled{opacity:.45;cursor:default}.pc2-status{display:inline-flex;align-items:center;gap:5px;padding:3px 7px;border:1px solid #d0d5dd;border-radius:999px;background:#fff;color:#475467;font-size:9px;font-weight:700;text-transform:uppercase}.pc2-status span{width:5px;height:5px;border-radius:50%;background:currentColor}.pc2-status.good{border-color:#bbf7d0;background:#f0fdf4;color:#15803d}.pc2-status.bad{border-color:#fecaca;background:#fff5f5;color:#b42318}.pc2-status.live{border-color:#bfdbfe;background:#eff6ff;color:#1d4ed8}.pc2-detail-head{min-height:78px;padding:11px 20px;border-bottom:1px solid #e4e7ec;background:#fff}.pc2-detail-title{min-width:0;display:flex;align-items:center;gap:13px}.pc2-detail-title>div{min-width:0}.pc2-detail-title p{margin:0;color:#667085;font-size:10px}.pc2-detail-title h1{max-width:700px;overflow:hidden;font-size:17px;text-overflow:ellipsis;white-space:nowrap}.pc2-detail-title h1+div{display:flex;align-items:center;gap:7px}.pc2-detail-title code{color:#667085;font-size:9px}.pc2-back{height:34px;padding:0 10px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;font-size:10px;cursor:pointer}.pc2-head-actions{display:flex;align-items:center;gap:7px}.pc2-metrics{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));margin:12px 20px;border:1px solid #e1e5ea;border-radius:10px;background:#fff;overflow:hidden}.pc2-metric{min-width:0;display:flex;flex-direction:column;gap:3px;padding:10px 12px;border-right:1px solid #eceef1}.pc2-metric:last-child{border-right:0}.pc2-metric span{color:#667085;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.05em}.pc2-metric strong{overflow:hidden;color:#101828;font:600 18px ui-monospace,SFMono-Regular,Menlo,monospace;text-overflow:ellipsis}.pc2-metric small{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-evidence-grid{min-height:0;display:grid;grid-template-columns:minmax(0,1fr) 370px;gap:12px;flex:1;padding:0 20px 18px}.pc2-trace-surface,.pc2-inspector{min-width:0;min-height:0;display:flex;flex-direction:column;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden}.pc2-trace-toolbar,.pc2-inspector-head{min-height:55px;display:flex;align-items:center;justify-content:space-between;gap:10px;padding:8px 11px;border-bottom:1px solid #eceef1}.pc2-trace-toolbar>div:first-child,.pc2-inspector-head>div{display:flex;flex-direction:column;gap:2px}.pc2-trace-toolbar strong,.pc2-inspector-head strong{color:#1d2939;font-size:12px}.pc2-trace-toolbar span,.pc2-inspector-head span{color:#98a2b3;font-size:9px}.pc2-toolbar-controls{display:flex!important;flex-direction:row!important;align-items:center;gap:6px}.pc2-toolbar-controls input,.pc2-toolbar-controls select{height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#344054;font-size:10px}.pc2-toolbar-controls input{width:170px;padding:0 8px}.pc2-toolbar-controls select{padding:0 24px 0 7px}.pc2-icon{width:30px;height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#475467;cursor:pointer}.pc2-segment{display:flex;border:1px solid #d0d5dd;border-radius:6px;overflow:hidden}.pc2-segment button{height:28px;padding:0 8px;border:0;border-right:1px solid #d0d5dd;background:#fff;color:#667085;font-size:9px;cursor:pointer}.pc2-segment button:last-child{border-right:0}.pc2-segment button.active{background:#eff6ff;color:#1d4ed8;font-weight:700}.pc2-turn-list,.pc2-inspector-body{min-height:0;flex:1;overflow:auto;scrollbar-gutter:stable}.pc2-turn-list{padding:11px 14px 24px}.pc2-tree{display:flex;flex-direction:column}.pc2-turn{display:grid;grid-template-columns:24px minmax(0,1fr);gap:7px;padding:0;border:0;background:transparent;text-align:left;cursor:pointer}.pc2-turn-axis{display:flex;flex-direction:column;align-items:center}.pc2-node{z-index:1;width:10px;height:10px;margin-top:14px;border:2px solid #fff;border-radius:50%;background:#94a3b8;box-shadow:0 0 0 1px #cbd5e1}.pc2-node.user{background:#2563eb}.pc2-node.agent{background:#10b981}.pc2-node.system{background:#f59e0b}.pc2-turn-axis i{width:1px;min-height:42px;flex:1;background:#d7dce2}.pc2-turn-card{min-width:0;margin-bottom:7px;padding:9px 10px;border:1px solid #e4e7ec;border-radius:8px;background:#fff}.pc2-turn:hover .pc2-turn-card,.pc2-turn:focus-visible .pc2-turn-card,.pc2-turn.active .pc2-turn-card{border-color:#93c5fd;box-shadow:0 0 0 3px #2563eb12}.pc2-turn:focus-visible{outline:0}.pc2-turn-top,.pc2-turn-meta{display:flex;align-items:center;gap:6px}.pc2-turn-top strong{color:#344054;font-size:10px}.pc2-turn-top code{color:#667085;font-size:9px}.pc2-grow{flex:1}.pc2-role{padding:2px 5px;border-radius:4px;background:#f2f4f7;color:#475467;font-size:8px;font-weight:800;text-transform:uppercase}.pc2-role.user{background:#eff6ff;color:#1d4ed8}.pc2-role.agent{background:#ecfdf5;color:#047857}.pc2-role.system{background:#fffbeb;color:#b45309}.pc2-error-chip{padding:2px 5px;border-radius:4px;background:#fff1f0;color:#b42318!important;font-size:8px}.pc2-turn-card>p{margin:7px 0;color:#475467;font:10px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-turn-meta{flex-wrap:wrap;color:#98a2b3;font-size:8px}.pc2-turn-meta code{margin-left:auto}.pc2-timeline-ruler,.pc2-time-row{display:grid;grid-template-columns:82px minmax(110px,1fr) 70px minmax(150px,1.5fr);gap:9px;align-items:center}.pc2-timeline-ruler{padding:0 7px 8px;color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-timeline-ruler span:nth-child(2){grid-column:2/4}.pc2-time-row{width:100%;min-height:38px;padding:5px 7px;border:0;border-top:1px solid #f0f1f3;background:#fff;color:#475467;text-align:left;cursor:pointer}.pc2-time-row:hover,.pc2-time-row.active{background:#f7faff}.pc2-time-label{display:flex;align-items:center;gap:5px;font-size:9px}.pc2-time-track{height:8px;border-radius:999px;background:#f0f2f5;overflow:hidden}.pc2-time-track i{display:block;height:100%;border-radius:999px;background:#60a5fa}.pc2-time-track i.bad{background:#ef4444}.pc2-time-row>code{color:#667085;font-size:9px}.pc2-time-preview{overflow:hidden;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-inspector-body{padding:10px}.pc2-inspector-facts{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1px;border:1px solid #e4e7ec;border-radius:7px;background:#e4e7ec;overflow:hidden}.pc2-inspector-facts>div{min-width:0;display:flex;flex-direction:column;gap:3px;padding:7px;background:#fff}.pc2-inspector-facts span{color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-inspector-facts code{overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis}.pc2-evidence-block{margin-top:9px;border:1px solid #e4e7ec;border-radius:7px;overflow:hidden}.pc2-evidence-block summary{padding:8px 9px;background:#f8fafc;color:#475467;font-size:9px;font-weight:700;cursor:pointer}.pc2-evidence-block pre{max-height:310px;margin:0;padding:9px;overflow:auto;border-top:1px solid #e4e7ec;background:#fff;color:#344054;font:9px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap;word-break:break-word}.pc2-inspector-empty,.pc2-copilot-empty{min-height:240px;display:flex;flex-direction:column;align-items:center;justify-content:center;padding:24px;color:#98a2b3;text-align:center}.pc2-inspector-empty>span{font-size:25px}.pc2-inspector-empty strong{margin-top:7px;color:#475467;font-size:12px}.pc2-inspector-empty p{max-width:260px;font-size:10px;line-height:1.6}.pc2-inline-loading,.pc2-loading,.pc2-chat-working{display:flex;align-items:center;justify-content:center;gap:7px;color:#667085;font-size:10px}.pc2-inline-loading{padding:10px}.pc2-loading{min-height:100%;flex-direction:column}.pc2-inline-loading .spinner,.pc2-chat-working .spinner{width:13px;height:13px;margin:0}.pc2-copilot{position:fixed;z-index:70;top:0;right:0;width:min(460px,42vw);height:100vh;display:flex;flex-direction:column;border-left:1px solid #1f2d40;background:#0d1726;color:#e5edf7;box-shadow:-16px 0 44px #1018282b}.pc2-copilot-head{min-height:62px;display:flex;align-items:center;justify-content:space-between;padding:10px 13px;border-bottom:1px solid #223046}.pc2-copilot-head>div:first-child{display:flex;flex-direction:column;gap:3px}.pc2-copilot-head strong{font-size:13px}.pc2-copilot-head span{color:#8291a5;font-size:9px}.pc2-copilot-head button,.pc2-settings header button{width:30px;height:30px;border:0;border-radius:6px;background:transparent;color:inherit;font-size:17px;cursor:pointer}.pc2-copilot-head button:hover{background:#ffffff0d}.pc2-context-card{margin:11px;padding:10px;border:1px solid #2b3a4f;border-radius:8px;background:#101e30}.pc2-context-card>div{display:flex;justify-content:space-between;gap:8px;margin-bottom:5px;font-size:9px}.pc2-context-card span{color:#8291a5}.pc2-context-card strong{max-width:260px;overflow:hidden;color:#cbd5e1;text-overflow:ellipsis;white-space:nowrap}.pc2-context-card label{display:flex;align-items:center;gap:6px;margin-top:8px;color:#9fb0c4;font-size:9px}.pc2-skill-chips{display:flex;flex-wrap:wrap;gap:5px;padding:0 11px 10px;border-bottom:1px solid #223046}.pc2-skill-chips button{padding:4px 7px;border:1px solid #33465f;border-radius:999px;background:#142236;color:#a9c7ef;font-size:8px;cursor:pointer}.pc2-chat{min-height:0;flex:1;overflow:auto;padding:12px;scrollbar-gutter:stable}.pc2-chat-welcome{display:flex;min-height:220px;flex-direction:column;align-items:center;justify-content:center;color:#718198;text-align:center}.pc2-chat-welcome>span{font-size:30px;color:#60a5fa}.pc2-chat-welcome strong{margin-top:8px;color:#dbeafe;font-size:12px}.pc2-chat-welcome p{max-width:330px;font-size:10px;line-height:1.6}.pc2-message{max-width:92%;margin-bottom:10px;padding:9px 10px;border-radius:10px;font-size:10px;line-height:1.55}.pc2-message.user{margin-left:auto;background:#2563eb;color:#fff}.pc2-message.assistant{border:1px solid #2b3a4f;background:#101e30;color:#d6e2f1}.pc2-action-label{display:inline-block;margin-bottom:5px;color:#60a5fa;font-size:8px;font-weight:800;text-transform:uppercase;letter-spacing:.07em}.pc2-message-text p{margin:0 0 6px;white-space:pre-wrap}.pc2-bullet{display:flex;gap:6px}.pc2-bullet p{flex:1}.pc2-message-sql{margin-top:7px;border:1px solid #33465f;border-radius:6px;overflow:hidden}.pc2-message-sql summary{padding:5px 7px;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-message-sql pre{max-height:180px;margin:0;padding:7px;overflow:auto;border-top:1px solid #33465f;background:#07101f;color:#bfdbfe;font:8px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap}.pc2-truncated{margin-top:7px;padding:5px 6px;border:1px solid #92400e;border-radius:5px;background:#451a0333;color:#fcd34d;font-size:8px}.pc2-citations{display:flex;flex-wrap:wrap;gap:5px;margin-top:7px}.pc2-citations button{padding:3px 6px;border:1px solid #3b82f6;border-radius:5px;background:#1d4ed822;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-composer{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;padding:10px;border-top:1px solid #223046}.pc2-composer textarea{min-height:62px;padding:8px;border:1px solid #33465f;border-radius:7px;outline:0;background:#101e30;color:#e5edf7;font:10px/1.45 inherit;resize:none}.pc2-composer textarea:focus{border-color:#3b82f6}.pc2-composer button{align-self:end}.pc2-modal-backdrop{position:fixed;z-index:90;inset:0;display:flex;align-items:stretch;justify-content:flex-end;background:#10182866;backdrop-filter:blur(2px)}.pc2-modal-backdrop.high{z-index:100;align-items:center;justify-content:center}.pc2-settings header{display:flex;align-items:center;justify-content:space-between;padding:17px;border-bottom:1px solid #e4e7ec}.pc2-settings h2{margin:2px 0;font-size:17px}.pc2-form{display:flex;flex-direction:column;gap:13px;padding:17px}.pc2-form label{display:flex;flex-direction:column;gap:5px;color:#475467;font-size:10px;font-weight:600}.pc2-form input,.pc2-form select,.pc2-form textarea{width:100%;padding:8px;border:1px solid #d0d5dd;border-radius:7px;outline:0;background:#fff;color:#344054;font-size:11px}.pc2-form textarea{min-height:130px;resize:vertical}.pc2-form input:focus,.pc2-form select:focus,.pc2-form textarea:focus{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb16}.pc2-form-error{padding:8px;border:1px solid #fecaca;border-radius:7px;background:#fff5f5;color:#b42318;font-size:9px}.pc2-settings footer{display:flex;justify-content:flex-end;gap:7px;padding:13px 17px;border-top:1px solid #e4e7ec}.pc2-settings{width:min(510px,calc(100vw - 30px));border-radius:11px;background:#fff;color:#344054;box-shadow:0 24px 80px #1018283d;overflow:hidden}.pc2-settings-note{margin:15px 17px 0;padding:9px;border:1px solid #bfdbfe;border-radius:7px;background:#eff6ff;color:#1e40af;font-size:9px;line-height:1.55}@media(max-width:1150px){.pc2-metrics{grid-template-columns:repeat(3,1fr)}.pc2-metric:nth-child(3){border-right:0}.pc2-metric:nth-child(-n+3){border-bottom:1px solid #eceef1}.pc2-evidence-grid{grid-template-columns:minmax(480px,1fr) 330px}.pc2-copilot{width:min(500px,55vw)}}@media(max-width:850px){.pc2-page{padding:16px}.pc2-filterbar{flex-wrap:wrap}.pc2-filter-search{min-width:100%;}.pc2-result-count{margin-left:0}.pc2-run-table{min-width:850px}.pc2-detail-head{align-items:flex-start}.pc2-head-actions a{display:none}.pc2-evidence-grid{grid-template-columns:1fr;overflow:auto}.pc2-trace-surface{min-height:520px}.pc2-inspector{min-height:420px}.pc2-copilot{width:calc(100vw - 56px);max-width:none}.pc2-toolbar-controls input{width:120px}} +.pc2-shell{height:100vh;display:grid;grid-template-columns:56px minmax(0,1fr);background:#f5f7fa;color:#172033}.pc2-main{position:relative;min-width:0;min-height:0;display:flex;flex-direction:column;overflow:hidden}.pc2-global-error{position:absolute;z-index:60;top:12px;left:50%;transform:translateX(-50%);display:flex;align-items:center;gap:10px;max-width:680px;padding:9px 12px;border:1px solid #fecaca;border-radius:9px;background:#fff7f7;color:#991b1b;font-size:11px;box-shadow:0 8px 24px #7f1d1d1a}.pc2-global-error button{margin-left:auto;border:0;background:transparent;color:inherit;font-size:18px;cursor:pointer}.pc2-page,.pc2-detail{min-height:0;display:flex;flex:1;flex-direction:column}.pc2-page{padding:22px 24px 18px;overflow:hidden}.pc2-page-head,.pc2-detail-head{display:flex;align-items:center;justify-content:space-between;gap:18px}.pc2-page-head{margin-bottom:18px}.pc2-page-head h1,.pc2-detail-head h1{margin:2px 0;color:#101828;font-size:22px;line-height:1.2}.pc2-page-head p:not(.eyebrow){margin:5px 0 0;color:#667085;font-size:12px}.pc2-filterbar{display:flex;align-items:center;gap:8px;margin-bottom:12px}.pc2-filterbar select,.pc2-filterbar button,.pc2-filter-search{height:36px;border:1px solid #d7dce3;border-radius:8px;background:#fff;color:#344054;font-size:11px}.pc2-filterbar select{padding:0 30px 0 10px}.pc2-filter-search{min-width:340px;display:flex;align-items:center;gap:7px;padding:0 10px;color:#98a2b3}.pc2-filter-search:focus-within{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb18}.pc2-filter-search input{min-width:0;flex:1;border:0;outline:0;background:transparent;color:#344054}.pc2-sort{padding:0 11px;cursor:pointer}.pc2-result-count{margin-left:auto;color:#667085;font-size:11px}.pc2-table-wrap{min-height:0;flex:1;border:1px solid #dfe3e8;border-radius:11px;background:#fff;overflow:auto;box-shadow:0 1px 2px #10182808}.pc2-run-table{width:100%;border-collapse:collapse;table-layout:fixed}.pc2-run-table th{position:sticky;z-index:2;top:0;padding:10px 12px;border-bottom:1px solid #e4e7ec;background:#f8fafc;color:#667085;text-align:left;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.07em}.pc2-run-table th:first-child{width:26%}.pc2-run-table th:nth-child(2){width:19%}.pc2-run-table th:nth-child(3){width:10%}.pc2-run-table th:nth-child(4),.pc2-run-table th:nth-child(5),.pc2-run-table th:nth-child(6){width:9%}.pc2-run-table td{height:61px;padding:9px 12px;border-bottom:1px solid #eef0f3;color:#475467;font-size:11px;vertical-align:middle}.pc2-run-table tbody tr{cursor:pointer}.pc2-run-table tbody tr:hover,.pc2-run-table tbody tr:focus-visible{outline:0;background:#f7faff;box-shadow:inset 3px 0 #3b82f6}.pc2-run-table td code{display:block;overflow:hidden;color:#667085;font-size:10px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell{min-width:0;display:flex;flex-direction:column;gap:4px}.pc2-session-cell strong{overflow:hidden;color:#1d2939;font-size:11px;text-overflow:ellipsis;white-space:nowrap}.pc2-session-cell span{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-number{font:600 11px ui-monospace,SFMono-Regular,Menlo,monospace}.pc2-table-skeleton td{height:61px;background:linear-gradient(90deg,#fafafa,#f0f3f7,#fafafa);background-size:200% 100%;animation:shimmer 1.4s infinite}.pc2-empty{min-height:180px;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:5px;color:#98a2b3;text-align:center}.pc2-empty strong{color:#475467;font-size:12px}.pc2-empty span{font-size:10px}.pc2-pagination{display:flex;align-items:center;justify-content:flex-end;gap:12px;padding-top:11px;color:#667085;font-size:10px}.pc2-pagination button{padding:6px 9px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;cursor:pointer}.pc2-pagination button:disabled{opacity:.45;cursor:default}.pc2-status{display:inline-flex;align-items:center;gap:5px;padding:3px 7px;border:1px solid #d0d5dd;border-radius:999px;background:#fff;color:#475467;font-size:9px;font-weight:700;text-transform:uppercase}.pc2-status span{width:5px;height:5px;border-radius:50%;background:currentColor}.pc2-status.good{border-color:#bbf7d0;background:#f0fdf4;color:#15803d}.pc2-status.bad{border-color:#fecaca;background:#fff5f5;color:#b42318}.pc2-status.live{border-color:#bfdbfe;background:#eff6ff;color:#1d4ed8}.pc2-detail-head{min-height:78px;padding:11px 20px;border-bottom:1px solid #e4e7ec;background:#fff}.pc2-detail-title{min-width:0;display:flex;align-items:center;gap:13px}.pc2-detail-title>div{min-width:0}.pc2-detail-title p{margin:0;color:#667085;font-size:10px}.pc2-detail-title h1{max-width:700px;overflow:hidden;font-size:17px;text-overflow:ellipsis;white-space:nowrap}.pc2-detail-title h1+div{display:flex;align-items:center;gap:7px}.pc2-detail-title code{color:#667085;font-size:9px}.pc2-back{height:34px;padding:0 10px;border:1px solid #d0d5dd;border-radius:7px;background:#fff;color:#344054;font-size:10px;cursor:pointer}.pc2-head-actions{display:flex;align-items:center;gap:7px}.pc2-metrics{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));margin:12px 20px;border:1px solid #e1e5ea;border-radius:10px;background:#fff;overflow:hidden}.pc2-metric{min-width:0;display:flex;flex-direction:column;gap:3px;padding:10px 12px;border-right:1px solid #eceef1}.pc2-metric:last-child{border-right:0}.pc2-metric span{color:#667085;font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.05em}.pc2-metric strong{overflow:hidden;color:#101828;font:600 18px ui-monospace,SFMono-Regular,Menlo,monospace;text-overflow:ellipsis}.pc2-metric small{overflow:hidden;color:#98a2b3;font-size:9px;text-overflow:ellipsis;white-space:nowrap}.pc2-trace-surface{min-width:0;min-height:0;display:flex;flex-direction:column;border:1px solid #dfe3e8;border-radius:10px;background:#fff;overflow:hidden}.pc2-trace-toolbar{min-height:55px;display:flex;align-items:center;justify-content:space-between;gap:10px;padding:8px 11px;border-bottom:1px solid #eceef1}.pc2-trace-toolbar>div:first-child{display:flex;flex-direction:column;gap:2px}.pc2-trace-toolbar strong{color:#1d2939;font-size:12px}.pc2-trace-toolbar span{color:#98a2b3;font-size:9px}.pc2-toolbar-controls{display:flex!important;flex-direction:row!important;align-items:center;gap:6px}.pc2-toolbar-controls input,.pc2-toolbar-controls select{height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#344054;font-size:10px}.pc2-toolbar-controls input{width:170px;padding:0 8px}.pc2-toolbar-controls select{padding:0 24px 0 7px}.pc2-icon{width:30px;height:30px;border:1px solid #d0d5dd;border-radius:6px;background:#fff;color:#475467;cursor:pointer}.pc2-turn-list{min-height:0;flex:1;overflow:auto;scrollbar-gutter:stable}.pc2-turn-list{padding:11px 14px 24px}.pc2-role{padding:2px 5px;border-radius:4px;background:#f2f4f7;color:#475467;font-size:8px;font-weight:800;text-transform:uppercase}.pc2-role.user{background:#eff6ff;color:#1d4ed8}.pc2-role.agent{background:#ecfdf5;color:#047857}.pc2-role.system{background:#fffbeb;color:#b45309}.pc2-error-chip{padding:2px 5px;border-radius:4px;background:#fff1f0;color:#b42318!important;font-size:8px}.pc2-inspector-facts{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1px;border:1px solid #e4e7ec;border-radius:7px;background:#e4e7ec;overflow:hidden}.pc2-inspector-facts>div{min-width:0;display:flex;flex-direction:column;gap:3px;padding:7px;background:#fff}.pc2-inspector-facts span{color:#98a2b3;font-size:8px;text-transform:uppercase}.pc2-inspector-facts code{overflow:hidden;color:#344054;font-size:9px;text-overflow:ellipsis}.pc2-evidence-block{margin-top:9px;border:1px solid #e4e7ec;border-radius:7px;overflow:hidden}.pc2-evidence-block summary{padding:8px 9px;background:#f8fafc;color:#475467;font-size:9px;font-weight:700;cursor:pointer}.pc2-evidence-block pre{max-height:310px;margin:0;padding:9px;overflow:auto;border-top:1px solid #e4e7ec;background:#fff;color:#344054;font:9px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap;word-break:break-word}.pc2-copilot-empty{min-height:240px;display:flex;flex-direction:column;align-items:center;justify-content:center;padding:24px;color:#98a2b3;text-align:center}.pc2-inline-loading,.pc2-loading,.pc2-chat-working{display:flex;align-items:center;justify-content:center;gap:7px;color:#667085;font-size:10px}.pc2-inline-loading{padding:10px}.pc2-loading{min-height:100%;flex-direction:column}.pc2-inline-loading .spinner,.pc2-chat-working .spinner{width:13px;height:13px;margin:0}.pc2-copilot{position:fixed;z-index:70;top:0;right:0;width:min(460px,42vw);height:100vh;display:flex;flex-direction:column;border-left:1px solid #1f2d40;background:#0d1726;color:#e5edf7;box-shadow:-16px 0 44px #1018282b}.pc2-copilot-head{min-height:62px;display:flex;align-items:center;justify-content:space-between;padding:10px 13px;border-bottom:1px solid #223046}.pc2-copilot-head>div:first-child{display:flex;flex-direction:column;gap:3px}.pc2-copilot-head strong{font-size:13px}.pc2-copilot-head span{color:#8291a5;font-size:9px}.pc2-copilot-head button,.pc2-settings header button{width:30px;height:30px;border:0;border-radius:6px;background:transparent;color:inherit;font-size:17px;cursor:pointer}.pc2-copilot-head button:hover{background:#ffffff0d}.pc2-context-card{margin:11px;padding:10px;border:1px solid #2b3a4f;border-radius:8px;background:#101e30}.pc2-context-card>div{display:flex;justify-content:space-between;gap:8px;margin-bottom:5px;font-size:9px}.pc2-context-card span{color:#8291a5}.pc2-context-card strong{max-width:260px;overflow:hidden;color:#cbd5e1;text-overflow:ellipsis;white-space:nowrap}.pc2-context-card label{display:flex;align-items:center;gap:6px;margin-top:8px;color:#9fb0c4;font-size:9px}.pc2-skill-chips{display:flex;flex-wrap:wrap;gap:5px;padding:0 11px 10px;border-bottom:1px solid #223046}.pc2-skill-chips button{padding:4px 7px;border:1px solid #33465f;border-radius:999px;background:#142236;color:#a9c7ef;font-size:8px;cursor:pointer}.pc2-chat{min-height:0;flex:1;overflow:auto;padding:12px;scrollbar-gutter:stable}.pc2-chat-welcome{display:flex;min-height:220px;flex-direction:column;align-items:center;justify-content:center;color:#718198;text-align:center}.pc2-chat-welcome>span{font-size:30px;color:#60a5fa}.pc2-chat-welcome strong{margin-top:8px;color:#dbeafe;font-size:12px}.pc2-chat-welcome p{max-width:330px;font-size:10px;line-height:1.6}.pc2-message{max-width:92%;margin-bottom:10px;padding:9px 10px;border-radius:10px;font-size:10px;line-height:1.55}.pc2-message.user{margin-left:auto;background:#2563eb;color:#fff}.pc2-message.assistant{border:1px solid #2b3a4f;background:#101e30;color:#d6e2f1}.pc2-action-label{display:inline-block;margin-bottom:5px;color:#60a5fa;font-size:8px;font-weight:800;text-transform:uppercase;letter-spacing:.07em}.pc2-message-text p{margin:0 0 6px;white-space:pre-wrap}.pc2-bullet{display:flex;gap:6px}.pc2-bullet p{flex:1}.pc2-message-sql{margin-top:7px;border:1px solid #33465f;border-radius:6px;overflow:hidden}.pc2-message-sql summary{padding:5px 7px;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-message-sql pre{max-height:180px;margin:0;padding:7px;overflow:auto;border-top:1px solid #33465f;background:#07101f;color:#bfdbfe;font:8px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap}.pc2-truncated{margin-top:7px;padding:5px 6px;border:1px solid #92400e;border-radius:5px;background:#451a0333;color:#fcd34d;font-size:8px}.pc2-citations{display:flex;flex-wrap:wrap;gap:5px;margin-top:7px}.pc2-citations button{padding:3px 6px;border:1px solid #3b82f6;border-radius:5px;background:#1d4ed822;color:#93c5fd;font-size:8px;cursor:pointer}.pc2-composer{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;padding:10px;border-top:1px solid #223046}.pc2-composer textarea{min-height:62px;padding:8px;border:1px solid #33465f;border-radius:7px;outline:0;background:#101e30;color:#e5edf7;font:10px/1.45 inherit;resize:none}.pc2-composer textarea:focus{border-color:#3b82f6}.pc2-composer button{align-self:end}.pc2-modal-backdrop{position:fixed;z-index:90;inset:0;display:flex;align-items:stretch;justify-content:flex-end;background:#10182866;backdrop-filter:blur(2px)}.pc2-modal-backdrop.high{z-index:100;align-items:center;justify-content:center}.pc2-settings header{display:flex;align-items:center;justify-content:space-between;padding:17px;border-bottom:1px solid #e4e7ec}.pc2-settings h2{margin:2px 0;font-size:17px}.pc2-form{display:flex;flex-direction:column;gap:13px;padding:17px}.pc2-form label{display:flex;flex-direction:column;gap:5px;color:#475467;font-size:10px;font-weight:600}.pc2-form input,.pc2-form select,.pc2-form textarea{width:100%;padding:8px;border:1px solid #d0d5dd;border-radius:7px;outline:0;background:#fff;color:#344054;font-size:11px}.pc2-form textarea{min-height:130px;resize:vertical}.pc2-form input:focus,.pc2-form select:focus,.pc2-form textarea:focus{border-color:#3b82f6;box-shadow:0 0 0 3px #2563eb16}.pc2-settings footer{display:flex;justify-content:flex-end;gap:7px;padding:13px 17px;border-top:1px solid #e4e7ec}.pc2-settings{width:min(510px,calc(100vw - 30px));border-radius:11px;background:#fff;color:#344054;box-shadow:0 24px 80px #1018283d;overflow:hidden}.pc2-settings-note{margin:15px 17px 0;padding:9px;border:1px solid #bfdbfe;border-radius:7px;background:#eff6ff;color:#1e40af;font-size:9px;line-height:1.55}@media(max-width:1150px){.pc2-metrics{grid-template-columns:repeat(3,1fr)}.pc2-metric:nth-child(3){border-right:0}.pc2-metric:nth-child(-n+3){border-bottom:1px solid #eceef1}.pc2-copilot{width:min(500px,55vw)}}@media(max-width:850px){.pc2-page{padding:16px}.pc2-filterbar{flex-wrap:wrap}.pc2-filter-search{min-width:100%;}.pc2-result-count{margin-left:0}.pc2-run-table{min-width:850px}.pc2-detail-head{align-items:flex-start}.pc2-head-actions a{display:none}.pc2-trace-surface{min-height:520px}.pc2-copilot{width:calc(100vw - 56px);max-width:none}.pc2-toolbar-controls input{width:120px}} From d82bdf231ac385607b9a2a3953f4d9af014ffd9c Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:15:05 +0800 Subject: [PATCH 10/65] refactor: replace pchronicle unreachable branches --- crates/persisting-pchronicle/src/formats/openai_corpus.rs | 8 +++++++- crates/persisting-pchronicle/src/projection/storyline.rs | 2 +- .../persisting-pchronicle/src/store/catalog/discovery.rs | 5 ++++- .../src/store/local_query_manifest.rs | 8 +++++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/persisting-pchronicle/src/formats/openai_corpus.rs b/crates/persisting-pchronicle/src/formats/openai_corpus.rs index 03a27016..3ee7e718 100644 --- a/crates/persisting-pchronicle/src/formats/openai_corpus.rs +++ b/crates/persisting-pchronicle/src/formats/openai_corpus.rs @@ -274,7 +274,13 @@ pub fn recover_openai_msg_files( envelope.insert("session_steps".into(), Value::Array(records)); Value::Object(envelope) } - _ => unreachable!("document kind validated above"), + kind => { + return Err(Error::Other(format!( + "invalid OpenAI document kind '{}' while recovering {}", + kind, + relative_path.display() + ))) + } }; output.push(RecoveredOpenaiMsgFile { relative_path, diff --git a/crates/persisting-pchronicle/src/projection/storyline.rs b/crates/persisting-pchronicle/src/projection/storyline.rs index c88290e2..bc32824b 100644 --- a/crates/persisting-pchronicle/src/projection/storyline.rs +++ b/crates/persisting-pchronicle/src/projection/storyline.rs @@ -183,7 +183,7 @@ pub async fn sync_storyline_projection( .. } = &previous.source else { - unreachable!() + anyhow::bail!("projection source is not canonical events; use `project rebuild`") }; if projection_lineage_is_fresh(&snapshot, previous) { diff --git a/crates/persisting-pchronicle/src/store/catalog/discovery.rs b/crates/persisting-pchronicle/src/store/catalog/discovery.rs index edc94293..915ae7d6 100644 --- a/crates/persisting-pchronicle/src/store/catalog/discovery.rs +++ b/crates/persisting-pchronicle/src/store/catalog/discovery.rs @@ -237,7 +237,10 @@ pub(super) fn bind_canonical_storyline_projections( continue; }; let LazySourceSpec::Events { snapshot, .. } = &events.spec else { - unreachable!() + anyhow::bail!( + "catalog source '{}' matched canonical event URI but is not an events source", + events.file + ) }; let last_modified = source_rows .iter() diff --git a/crates/persisting-pchronicle/src/store/local_query_manifest.rs b/crates/persisting-pchronicle/src/store/local_query_manifest.rs index d81c91ed..c2902a57 100644 --- a/crates/persisting-pchronicle/src/store/local_query_manifest.rs +++ b/crates/persisting-pchronicle/src/store/local_query_manifest.rs @@ -162,7 +162,13 @@ impl LocalQueryManifest { let extensions: &[&str] = match format { ChronicleFormat::Atif => &["json", "jsonl", "ndjson"], ChronicleFormat::OpenaiMsg | ChronicleFormat::Actf => &["json"], - _ => unreachable!("query format was validated above"), + _ => { + anyhow::bail!( + "unsupported direct query format '{}' in {}", + format, + input.display() + ) + } }; let paths = input_files_with_extensions(input, extensions, options)?; anyhow::ensure!( From ab7e9d8f8cfe716b2d5eabc40877c88abc382dd6 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:18:20 +0800 Subject: [PATCH 11/65] refactor: colocate storyline row model --- crates/persisting-pchronicle/src/lib.rs | 10 ++++------ crates/persisting-pchronicle/src/store/mod.rs | 6 ++++++ .../src/store/storyline/mod.rs | 4 ++-- .../storyline/model.rs} | 0 .../src/store/storyline/rows.rs | 2 +- .../2026-08-17-pchronicle-post-judge-cleanup.md | 13 ++++++------- ...26-08-17-pchronicle-post-judge-cleanup-design.md | 5 ++++- 7 files changed, 23 insertions(+), 17 deletions(-) rename crates/persisting-pchronicle/src/{storyline_schema.rs => store/storyline/model.rs} (100%) diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index 1cdf281c..1fb7a8a8 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -41,7 +41,6 @@ pub mod revision; #[cfg(feature = "search")] pub mod search; pub mod store; -pub mod storyline_schema; #[cfg(feature = "lance-store")] pub use append_queue::{ @@ -132,7 +131,9 @@ pub use store::{ find_block_by_call_id_and_role, index_agenticmd_path, list_agenticmd_paths, parse_agenticmd_document_validated, parse_agenticmd_spans_validated, read_agenticmd_blocks_from_file, rewrite_agenticmd_preamble, rewrite_block_range, - upsert_block_by_call_id, write_agenticmd_document, AgenticmdFileIndex, + upsert_block_by_call_id, write_agenticmd_document, AgenticmdFileIndex, StoryRunRow, + StoryStepRow, StoryToolCallRow, StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, + STORY_TOOL_CALLS_TABLE, }; #[cfg(feature = "lance-store")] pub use store::{ @@ -171,10 +172,7 @@ pub use store::{ }; #[cfg(feature = "lance-store")] pub use store::{detect_local_query_format, detect_local_query_manifest}; -pub use storyline_schema::{ - reconstruct_storyline, split_storyline, StoryRunRow, StoryStepRow, StoryToolCallRow, - StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, STORY_TOOL_CALLS_TABLE, -}; +pub use store::{reconstruct_storyline, split_storyline}; #[cfg(feature = "search")] pub const PERSISTING_VECTOR_INDEX_NAME: &str = search::search_lance::PERSISTING_VECTOR_INDEX_NAME; diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 26b207c1..90834249 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -36,6 +36,8 @@ mod root_write_lock; mod run_control; #[cfg(feature = "lance-store")] mod storyline; +#[path = "storyline/model.rs"] +mod storyline_model; pub use agenticmd_fs::{ agenticmd_block_count, agenticmd_replay_json_lines, agenticmd_structural_issues, @@ -108,6 +110,10 @@ pub use storyline::{ DATAFUSION_STEPS_TABLE, DATAFUSION_TOOL_CALLS_TABLE, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, }; +pub use storyline_model::{ + reconstruct_storyline, split_storyline, StoryRunRow, StoryStepRow, StoryToolCallRow, + StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, STORY_TOOL_CALLS_TABLE, +}; #[cfg(feature = "lance-store")] use std::path::PathBuf; diff --git a/crates/persisting-pchronicle/src/store/storyline/mod.rs b/crates/persisting-pchronicle/src/store/storyline/mod.rs index c8f3e123..f0d0836e 100644 --- a/crates/persisting-pchronicle/src/store/storyline/mod.rs +++ b/crates/persisting-pchronicle/src/store/storyline/mod.rs @@ -68,11 +68,11 @@ use object_store::path::Path as ObjectPath; use object_store::{Error as ObjectStoreError, ObjectStoreExt, PutMode, UpdateVersion}; use serde::{Deserialize, Serialize}; -use crate::convert::atif_to_storyline; -use crate::storyline_schema::{ +use super::storyline_model::{ reconstruct_storyline, split_storyline, StoryRunRow, StoryStepRow, StoryToolCallRow, StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, STORY_TOOL_CALLS_TABLE, }; +use crate::convert::atif_to_storyline; use crate::StorylineDocument; use self::content::{ diff --git a/crates/persisting-pchronicle/src/storyline_schema.rs b/crates/persisting-pchronicle/src/store/storyline/model.rs similarity index 100% rename from crates/persisting-pchronicle/src/storyline_schema.rs rename to crates/persisting-pchronicle/src/store/storyline/model.rs diff --git a/crates/persisting-pchronicle/src/store/storyline/rows.rs b/crates/persisting-pchronicle/src/store/storyline/rows.rs index 0857a5b6..1b46ac62 100644 --- a/crates/persisting-pchronicle/src/store/storyline/rows.rs +++ b/crates/persisting-pchronicle/src/store/storyline/rows.rs @@ -10,7 +10,7 @@ use lance::deps::arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit use serde::de::DeserializeOwned; use serde::Serialize; -use crate::storyline_schema::{StoryRunRow, StoryStepRow, StoryToolCallRow}; +use super::super::storyline_model::{StoryRunRow, StoryStepRow, StoryToolCallRow}; fn field(name: &str, data_type: DataType, nullable: bool) -> Field { Field::new(name, data_type, nullable) diff --git a/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md b/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md index 42ad0173..c3a0c9af 100644 --- a/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md +++ b/docs/superpowers/plans/2026-08-17-pchronicle-post-judge-cleanup.md @@ -171,24 +171,23 @@ cargo test -p persisting-pchronicle storyline_schema::tests --locked - [ ] **步骤 2：移动逻辑模型并改为同目录引用** -将文件移动为 `store/storyline/model.rs`；在 `store/storyline/mod.rs` 声明 -`mod model;`，并从本地 `model` 导入类型与函数。`rows.rs` 使用 -`super::model::{StoryRunRow, StoryStepRow, StoryToolCallRow}`。 +将文件移动为 `store/storyline/model.rs`。为保留无 Lance feature 时的根级 API， +在 `store/mod.rs` 用 `#[path = "storyline/model.rs"]` 注册不受 feature 门控的 +`storyline_model`；`store/storyline/mod.rs` 与 `rows.rs` 从该兄弟模块导入。 - [ ] **步骤 3：保持根级 item 导出并删除根模块** -由 `store/storyline/mod.rs` re-export： +由 `store/mod.rs` re-export： ```rust -pub use model::{ +pub use storyline_model::{ reconstruct_storyline, split_storyline, StoryRunRow, StoryStepRow, StoryToolCallRow, StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, STORY_TOOL_CALLS_TABLE, }; ``` -`store/mod.rs` 将这些 item 向上 re-export；`lib.rs` 从 `store` 导出它们并删除 -`pub mod storyline_schema` 及对应导出块。 +`lib.rs` 从 `store` 导出这些 item，并删除 `pub mod storyline_schema` 及对应导出块。 - [ ] **步骤 4：运行迁移后的测试与编译** diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md index af4d9732..0c0058a0 100644 --- a/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md +++ b/docs/superpowers/specs/2026-08-17-pchronicle-post-judge-cleanup-design.md @@ -104,7 +104,10 @@ Storyline 三表逻辑模型从 crate 根部的 `storyline_schema.rs` 移至 crate 根继续导出 `split_storyline`、`reconstruct_storyline`、`StoryRunRow`、 `StoryStepRow`、`StoryToolCallRow`、`StorylineTables` 和三个表名常量。删除根级 -`storyline_schema` 模块，Storyline 存储内部通过同目录模块导入逻辑模型。 +`storyline_schema` 模块。为保证这些 item 在 `--no-default-features` 下仍然可用， +`store/mod.rs` 在不受 `lance-store` 门控的位置注册物理上同目录的模型模块；受 +feature 门控的 Storyline 存储和 Arrow codec 从该兄弟模块导入。这样同时保留 +feature 边界与目录收拢，不把纯逻辑模型错误地绑定到 Lance。 ## 验收与验证 From 2e93bbb33bd14608c1cc9f367c68d18e9fc4ddf3 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:19:58 +0800 Subject: [PATCH 12/65] fix: declare gateway reqwest json feature --- crates/persisting-gateway/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/persisting-gateway/Cargo.toml b/crates/persisting-gateway/Cargo.toml index 826a0ad1..2e3c3b54 100644 --- a/crates/persisting-gateway/Cargo.toml +++ b/crates/persisting-gateway/Cargo.toml @@ -19,7 +19,7 @@ bytes.workspace = true chrono = { workspace = true, features = ["serde"] } http-body-util.workspace = true futures-util = { workspace = true, features = ["std"] } -reqwest = { workspace = true, features = ["rustls-tls", "stream"] } +reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } serde = { workspace = true, features = ["derive"] } serde_json.workspace = true toml.workspace = true From d95b83335755c9eb025b04be80d2551d9fcafcb7 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:26:15 +0800 Subject: [PATCH 13/65] refactor: consolidate agenticmd domain --- .../agenticmd_body.rs => agenticmd/body.rs} | 0 .../agenticmd.rs => agenticmd/codec.rs} | 0 .../agenticmd.rs => agenticmd/convert.rs} | 7 +-- .../frontmatter.rs} | 4 +- .../agenticmd_fs.rs => agenticmd/fs.rs} | 14 ++--- .../markdown.rs => agenticmd/layout.rs} | 0 .../src/{ => agenticmd}/mapping/fields.rs | 0 .../src/{ => agenticmd}/mapping/mod.rs | 9 ++-- .../src/{ => agenticmd}/mapping/text.rs | 0 .../src/agenticmd/mod.rs | 52 +++++++++++++++++++ .../agenticmd.rs => agenticmd/projection.rs} | 17 +++--- .../validate.rs} | 2 +- .../persisting-pchronicle/src/convert/mod.rs | 3 +- .../persisting-pchronicle/src/formats/mod.rs | 22 ++++---- .../persisting-pchronicle/src/layout/mod.rs | 5 +- .../src/layout/resolve.rs | 2 +- crates/persisting-pchronicle/src/lib.rs | 14 ++--- .../src/projection/mod.rs | 3 +- crates/persisting-pchronicle/src/store/mod.rs | 3 +- crates/persisting-pchronicle/src/tests.rs | 4 +- 20 files changed, 105 insertions(+), 56 deletions(-) rename crates/persisting-pchronicle/src/{formats/agenticmd_body.rs => agenticmd/body.rs} (100%) rename crates/persisting-pchronicle/src/{formats/agenticmd.rs => agenticmd/codec.rs} (100%) rename crates/persisting-pchronicle/src/{convert/agenticmd.rs => agenticmd/convert.rs} (99%) rename crates/persisting-pchronicle/src/{formats/agenticmd_frontmatter.rs => agenticmd/frontmatter.rs} (96%) rename crates/persisting-pchronicle/src/{store/agenticmd_fs.rs => agenticmd/fs.rs} (98%) rename crates/persisting-pchronicle/src/{layout/markdown.rs => agenticmd/layout.rs} (100%) rename crates/persisting-pchronicle/src/{ => agenticmd}/mapping/fields.rs (100%) rename crates/persisting-pchronicle/src/{ => agenticmd}/mapping/mod.rs (96%) rename crates/persisting-pchronicle/src/{ => agenticmd}/mapping/text.rs (100%) create mode 100644 crates/persisting-pchronicle/src/agenticmd/mod.rs rename crates/persisting-pchronicle/src/{projection/agenticmd.rs => agenticmd/projection.rs} (92%) rename crates/persisting-pchronicle/src/{formats/agenticmd_validate.rs => agenticmd/validate.rs} (96%) diff --git a/crates/persisting-pchronicle/src/formats/agenticmd_body.rs b/crates/persisting-pchronicle/src/agenticmd/body.rs similarity index 100% rename from crates/persisting-pchronicle/src/formats/agenticmd_body.rs rename to crates/persisting-pchronicle/src/agenticmd/body.rs diff --git a/crates/persisting-pchronicle/src/formats/agenticmd.rs b/crates/persisting-pchronicle/src/agenticmd/codec.rs similarity index 100% rename from crates/persisting-pchronicle/src/formats/agenticmd.rs rename to crates/persisting-pchronicle/src/agenticmd/codec.rs diff --git a/crates/persisting-pchronicle/src/convert/agenticmd.rs b/crates/persisting-pchronicle/src/agenticmd/convert.rs similarity index 99% rename from crates/persisting-pchronicle/src/convert/agenticmd.rs rename to crates/persisting-pchronicle/src/agenticmd/convert.rs index 88486e72..7a5fdd83 100644 --- a/crates/persisting-pchronicle/src/convert/agenticmd.rs +++ b/crates/persisting-pchronicle/src/agenticmd/convert.rs @@ -8,12 +8,13 @@ use std::collections::BTreeMap; use serde_json::{json, Map, Value}; use crate::convert::message_text; -use crate::formats::agenticmd::{ +use crate::formats::storyline::{StorylineAgent, StorylineDocument, StorylineTurn}; +use crate::Result; + +use super::codec::{ AgenticmdBlock, AgenticmdDocument, AgenticmdHeader, AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, }; -use crate::formats::storyline::{StorylineAgent, StorylineDocument, StorylineTurn}; -use crate::Result; /// Header field names preserved via `turn.extra` for hub round-trips. const EXTRA_CORRELATION_KEYS: &[&str] = &[ diff --git a/crates/persisting-pchronicle/src/formats/agenticmd_frontmatter.rs b/crates/persisting-pchronicle/src/agenticmd/frontmatter.rs similarity index 96% rename from crates/persisting-pchronicle/src/formats/agenticmd_frontmatter.rs rename to crates/persisting-pchronicle/src/agenticmd/frontmatter.rs index 3a266463..24cda7ac 100644 --- a/crates/persisting-pchronicle/src/formats/agenticmd_frontmatter.rs +++ b/crates/persisting-pchronicle/src/agenticmd/frontmatter.rs @@ -2,7 +2,9 @@ use serde::{Deserialize, Serialize}; -use super::{encode_agenticmd_preamble, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT}; +use super::codec::{ + encode_agenticmd_preamble, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, +}; use crate::Result; /// Producer/client provenance embedded in an AgenticMD document. diff --git a/crates/persisting-pchronicle/src/store/agenticmd_fs.rs b/crates/persisting-pchronicle/src/agenticmd/fs.rs similarity index 98% rename from crates/persisting-pchronicle/src/store/agenticmd_fs.rs rename to crates/persisting-pchronicle/src/agenticmd/fs.rs index ba426b3d..a1966456 100644 --- a/crates/persisting-pchronicle/src/store/agenticmd_fs.rs +++ b/crates/persisting-pchronicle/src/agenticmd/fs.rs @@ -8,15 +8,15 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use serde::Serialize; -use crate::formats::agenticmd::{ +use super::codec::{ encode_agenticmd_block, encode_agenticmd_preamble, parse_agenticmd_blocks_with_spans, parse_agenticmd_document, AgenticmdBlock, AgenticmdBlockSpan, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, }; -use crate::formats::agenticmd_validate::{ +use super::mapping::agenticmd_block_to_replay_json; +use super::validate::{ block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name, }; -use crate::mapping::agenticmd_block_to_replay_json; /// Diagnostic index of one AgenticMD document. #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -130,7 +130,7 @@ pub fn write_agenticmd_document( pub fn rewrite_agenticmd_preamble(path: &Path, preamble: &str) -> Result<()> { let content = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; - let body_start = crate::formats::agenticmd_body_byte_offset(&content) + let body_start = super::codec::agenticmd_body_byte_offset(&content) .map_err(|e| anyhow::anyhow!("agenticmd body offset: {e}"))?; let mut output = preamble.as_bytes().to_vec(); output.extend_from_slice(&content.as_bytes()[body_start..]); @@ -334,11 +334,11 @@ fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> { #[cfg(test)] mod tests { - use super::*; - use crate::formats::agenticmd::{ + use super::super::codec::{ encode_agenticmd_preamble, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, }; + use super::*; use serde::Serialize; use serde_json::json; use std::collections::BTreeMap; @@ -570,7 +570,7 @@ mod tests { body: body.into(), }) .unwrap(); - let via_raw = crate::formats::agenticmd::encode_agenticmd_block(&AgenticmdBlock { + let via_raw = super::super::codec::encode_agenticmd_block(&AgenticmdBlock { header: AgenticmdHeader { type_name: "text".into(), length: body.len(), diff --git a/crates/persisting-pchronicle/src/layout/markdown.rs b/crates/persisting-pchronicle/src/agenticmd/layout.rs similarity index 100% rename from crates/persisting-pchronicle/src/layout/markdown.rs rename to crates/persisting-pchronicle/src/agenticmd/layout.rs diff --git a/crates/persisting-pchronicle/src/mapping/fields.rs b/crates/persisting-pchronicle/src/agenticmd/mapping/fields.rs similarity index 100% rename from crates/persisting-pchronicle/src/mapping/fields.rs rename to crates/persisting-pchronicle/src/agenticmd/mapping/fields.rs diff --git a/crates/persisting-pchronicle/src/mapping/mod.rs b/crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs similarity index 96% rename from crates/persisting-pchronicle/src/mapping/mod.rs rename to crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs index 1a51c029..37a324e4 100644 --- a/crates/persisting-pchronicle/src/mapping/mod.rs +++ b/crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs @@ -9,12 +9,11 @@ mod text; use anyhow::{Context, Result}; use serde_json::{json, Value}; -use crate::formats::agenticmd::{AgenticmdBlock, AgenticmdHeader}; -use crate::formats::agenticmd_body::{ - append_subagent_refs_footer, strip_subagent_footer_from_body, -}; use crate::formats::events::{EventIdentity, EventRecord}; +use super::body::{append_subagent_refs_footer, strip_subagent_footer_from_body}; +use super::codec::{AgenticmdBlock, AgenticmdHeader}; + use fields::{attach_llm_fields, attach_subagent_link_fields, role_and_body}; /// Build an agenticmd block from an event record (primary write mapping). @@ -232,6 +231,6 @@ pub fn agenticmd_blocks_to_event_records(blocks: &[AgenticmdBlock]) -> Result Result> { - let parsed = crate::formats::agenticmd::parse_agenticmd_document(doc)?; + let parsed = super::codec::parse_agenticmd_document(doc)?; agenticmd_blocks_to_event_records(&parsed.blocks) } diff --git a/crates/persisting-pchronicle/src/mapping/text.rs b/crates/persisting-pchronicle/src/agenticmd/mapping/text.rs similarity index 100% rename from crates/persisting-pchronicle/src/mapping/text.rs rename to crates/persisting-pchronicle/src/agenticmd/mapping/text.rs diff --git a/crates/persisting-pchronicle/src/agenticmd/mod.rs b/crates/persisting-pchronicle/src/agenticmd/mod.rs new file mode 100644 index 00000000..5d2f3013 --- /dev/null +++ b/crates/persisting-pchronicle/src/agenticmd/mod.rs @@ -0,0 +1,52 @@ +//! AgenticMD debug-view domain: codec, mapping, paths, filesystem I/O, and projections. + +mod body; +mod codec; +mod convert; +mod frontmatter; +mod fs; +mod layout; +mod mapping; +#[cfg(feature = "lance-store")] +mod projection; +mod validate; + +pub use body::{ + append_subagent_refs_footer, is_subagent_footer_line, strip_subagent_footer_from_body, +}; +pub use codec::{ + agenticmd_body_byte_offset, encode_agenticmd_block, encode_agenticmd_document, + encode_agenticmd_preamble, parse_agenticmd_blocks_with_spans, parse_agenticmd_document, + AgenticmdBlock, AgenticmdBlockSpan, AgenticmdDocument, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, + AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, +}; +pub use convert::{agenticmd_to_storyline, storyline_to_agenticmd}; +pub use frontmatter::{ + encode_agenticmd_session_frontmatter, AgenticmdClientMeta, AgenticmdSessionFrontmatter, +}; +pub use fs::{ + agenticmd_block_count, agenticmd_replay_json_lines, agenticmd_structural_issues, + append_agenticmd_blocks, count_agenticmd_role, encode_agenticmd_block_validated, + find_block_by_call_id_and_role, index_agenticmd_path, list_agenticmd_paths, + parse_agenticmd_document_validated, parse_agenticmd_spans_validated, + read_agenticmd_blocks_from_file, rewrite_agenticmd_preamble, rewrite_block_range, + upsert_block_by_call_id, write_agenticmd_document, AgenticmdFileIndex, +}; +pub use layout::{ + is_subagent_session_storage_key, is_trajectory_markdown_path, locate_run_bucket_markdown, + locate_session_markdown, locate_session_markdown_for_key, sanitize_session_filename, + session_markdown_filename, session_markdown_path_for_key, session_markdown_write_path_for_key, +}; +pub use mapping::{ + agenticmd_block_to_event_record, agenticmd_block_to_replay_json, + agenticmd_blocks_to_event_records, enrich_event_from_agenticmd_block, + event_record_to_agenticmd_block, event_record_to_agenticmd_block_with_text, + markdown_document_to_event_records, +}; +#[cfg(feature = "lance-store")] +pub use projection::{ + event_records_to_markdown_blocks, layer_stats, materialize_lance_to_markdown, + materialize_markdown_path, write_markdown_projection, LayerStats, MaterializeOutcome, + MaterializeStats, +}; +pub use validate::{block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name}; diff --git a/crates/persisting-pchronicle/src/projection/agenticmd.rs b/crates/persisting-pchronicle/src/agenticmd/projection.rs similarity index 92% rename from crates/persisting-pchronicle/src/projection/agenticmd.rs rename to crates/persisting-pchronicle/src/agenticmd/projection.rs index a63f653e..60a1ebb0 100644 --- a/crates/persisting-pchronicle/src/projection/agenticmd.rs +++ b/crates/persisting-pchronicle/src/agenticmd/projection.rs @@ -5,11 +5,14 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; -use crate::{ - agenticmd_block_count, encode_agenticmd_preamble, locate_session_markdown_for_key, - session_markdown_path_for_key, write_agenticmd_document, EventRecord, RawEventLanceStore, - StoryCoords, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, +use crate::{EventRecord, RawEventLanceStore, StoryCoords}; + +use super::codec::{ + encode_agenticmd_preamble, AgenticmdBlock, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, }; +use super::fs::{agenticmd_block_count, write_agenticmd_document}; +use super::layout::{locate_session_markdown_for_key, session_markdown_path_for_key}; +use super::mapping::event_record_to_agenticmd_block; #[derive(Debug, Clone, PartialEq, Eq)] pub struct MaterializeStats { @@ -47,7 +50,7 @@ pub fn materialize_markdown_path(run_dir: &Path, session_key: &str) -> PathBuf { pub fn event_records_to_markdown_blocks( records: &[EventRecord], -) -> Result<(Vec, MaterializeStats)> { +) -> Result<(Vec, MaterializeStats)> { let blocks = project_dialogue_blocks(records)?; let stats = MaterializeStats { source_events: records.len(), @@ -62,7 +65,7 @@ pub fn event_records_to_markdown_blocks( /// This deliberately drops transport-only and duplicate streaming events. /// AgenticMD is not a persistence boundary, so callers must retain the source /// events or Storyline document when lossless replay is required. -fn project_dialogue_blocks(records: &[EventRecord]) -> Result> { +fn project_dialogue_blocks(records: &[EventRecord]) -> Result> { let mut last_user_message_count = 0usize; let mut skipped_call_ids = HashSet::new(); let mut blocks = Vec::new(); @@ -111,7 +114,7 @@ fn project_dialogue_blocks(records: &[EventRecord]) -> Result false, }; if !skip { - let block = crate::event_record_to_agenticmd_block(record)?; + let block = event_record_to_agenticmd_block(record)?; if !block.body.trim().is_empty() || record.kind == "llm.spawn_link" { blocks.push(block); } diff --git a/crates/persisting-pchronicle/src/formats/agenticmd_validate.rs b/crates/persisting-pchronicle/src/agenticmd/validate.rs similarity index 96% rename from crates/persisting-pchronicle/src/formats/agenticmd_validate.rs rename to crates/persisting-pchronicle/src/agenticmd/validate.rs index 8e7dc5f7..7a684afd 100644 --- a/crates/persisting-pchronicle/src/formats/agenticmd_validate.rs +++ b/crates/persisting-pchronicle/src/agenticmd/validate.rs @@ -5,7 +5,7 @@ use anyhow::{bail, Result}; -use super::agenticmd::{AgenticmdBlock, AgenticmdHeader}; +use super::codec::{AgenticmdBlock, AgenticmdHeader}; pub fn block_speaker(header: &AgenticmdHeader) -> &str { header diff --git a/crates/persisting-pchronicle/src/convert/mod.rs b/crates/persisting-pchronicle/src/convert/mod.rs index 4fa412c5..cf2792fb 100644 --- a/crates/persisting-pchronicle/src/convert/mod.rs +++ b/crates/persisting-pchronicle/src/convert/mod.rs @@ -13,15 +13,14 @@ //! [`storyline_to_events`] after loading Lance rows into [`EventsDocument`]. mod actf; -mod agenticmd; mod atif; mod events; mod openai_msg; +pub use crate::agenticmd::{agenticmd_to_storyline, storyline_to_agenticmd}; pub use actf::{ actf_to_storyline, actf_to_storylines, is_actf_storyline, storyline_to_actf, storylines_to_actf, }; -pub use agenticmd::{agenticmd_to_storyline, storyline_to_agenticmd}; pub use atif::{atif_to_storyline, storyline_to_atif}; #[cfg(feature = "lance-store")] pub(crate) use events::event_storyline_key; diff --git a/crates/persisting-pchronicle/src/formats/mod.rs b/crates/persisting-pchronicle/src/formats/mod.rs index c88b8dfd..4901030e 100644 --- a/crates/persisting-pchronicle/src/formats/mod.rs +++ b/crates/persisting-pchronicle/src/formats/mod.rs @@ -1,10 +1,6 @@ //! Codecs for each [`crate::ChronicleFormat`]. pub mod actf; -pub mod agenticmd; -pub mod agenticmd_body; -pub mod agenticmd_frontmatter; -pub mod agenticmd_validate; pub mod detect; pub mod events; pub mod llm; @@ -12,24 +8,24 @@ pub mod openai_corpus; pub mod openai_msg; pub mod storyline; -pub use actf::{ - parse_actf_document, ActfAssistantContent, ActfAttempt, ActfDocument, ActfMetric, - ActfObservation, ActfStep, ActfToolCall, ActfTrajectory, ACTF_SCHEMA_VERSION, -}; -pub use agenticmd::{ +pub use crate::agenticmd::{ agenticmd_body_byte_offset, encode_agenticmd_block, encode_agenticmd_document, encode_agenticmd_preamble, parse_agenticmd_blocks_with_spans, parse_agenticmd_document, AgenticmdBlock, AgenticmdBlockSpan, AgenticmdDocument, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, }; -pub use agenticmd_body::{ +pub use crate::agenticmd::{ append_subagent_refs_footer, is_subagent_footer_line, strip_subagent_footer_from_body, }; -pub use agenticmd_frontmatter::{ +pub use crate::agenticmd::{ + block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name, +}; +pub use crate::agenticmd::{ encode_agenticmd_session_frontmatter, AgenticmdClientMeta, AgenticmdSessionFrontmatter, }; -pub use agenticmd_validate::{ - block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name, +pub use actf::{ + parse_actf_document, ActfAssistantContent, ActfAttempt, ActfDocument, ActfMetric, + ActfObservation, ActfStep, ActfToolCall, ActfTrajectory, ACTF_SCHEMA_VERSION, }; pub use detect::detect_format; pub use events::{ diff --git a/crates/persisting-pchronicle/src/layout/mod.rs b/crates/persisting-pchronicle/src/layout/mod.rs index 876a20cf..60b8e140 100644 --- a/crates/persisting-pchronicle/src/layout/mod.rs +++ b/crates/persisting-pchronicle/src/layout/mod.rs @@ -1,15 +1,14 @@ //! On-disk layout for agent trajectory storage (run dirs, markdown, Lance paths). mod coords; -mod markdown; mod resolve; -pub use coords::{story_lance_event_path, story_run_dir, StoryCoords}; -pub use markdown::{ +pub use crate::agenticmd::{ is_subagent_session_storage_key, is_trajectory_markdown_path, locate_run_bucket_markdown, locate_session_markdown, locate_session_markdown_for_key, sanitize_session_filename, session_markdown_filename, session_markdown_path_for_key, session_markdown_write_path_for_key, }; +pub use coords::{story_lance_event_path, story_run_dir, StoryCoords}; pub use resolve::{ list_story_read_locations, merge_story_location, resolve_story_read_location, try_infer_story_location, StoryLocationPartial, diff --git a/crates/persisting-pchronicle/src/layout/resolve.rs b/crates/persisting-pchronicle/src/layout/resolve.rs index d4f38f6a..0dee373d 100644 --- a/crates/persisting-pchronicle/src/layout/resolve.rs +++ b/crates/persisting-pchronicle/src/layout/resolve.rs @@ -8,7 +8,7 @@ use std::path::{Component, Path}; use anyhow::{Context, Result}; use super::coords::StoryCoords; -use super::markdown::{locate_session_markdown, session_markdown_path_for_key}; +use crate::agenticmd::{locate_session_markdown, session_markdown_path_for_key}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct StoryLocationPartial { diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index 1fb7a8a8..aa813b02 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -19,6 +19,7 @@ //! //! Use [`convert::into_storyline`] / [`convert::from_storyline`] / [`convert::convert`]. +mod agenticmd; #[cfg(feature = "lance-store")] mod append_queue; pub mod atif; @@ -30,7 +31,6 @@ pub mod format; pub mod formats; pub mod interop; pub mod layout; -pub mod mapping; mod messages; #[cfg(feature = "lance-store")] pub mod operations; @@ -42,6 +42,12 @@ pub mod revision; pub mod search; pub mod store; +pub use agenticmd::{ + agenticmd_block_to_event_record, agenticmd_block_to_replay_json, + agenticmd_blocks_to_event_records, enrich_event_from_agenticmd_block, + event_record_to_agenticmd_block, event_record_to_agenticmd_block_with_text, + markdown_document_to_event_records, +}; #[cfg(feature = "lance-store")] pub use append_queue::{ raw_event_append_queue, raw_event_append_queue_with_capacity, RawEventAppendQueueError, @@ -95,12 +101,6 @@ pub use layout::{ story_lance_event_path, story_run_dir, try_infer_story_location, StoryCoords, StoryLocationPartial, }; -pub use mapping::{ - agenticmd_block_to_event_record, agenticmd_block_to_replay_json, - agenticmd_blocks_to_event_records, enrich_event_from_agenticmd_block, - event_record_to_agenticmd_block, event_record_to_agenticmd_block_with_text, - markdown_document_to_event_records, -}; pub use messages::*; #[cfg(feature = "search")] pub use operations::bridge::{ diff --git a/crates/persisting-pchronicle/src/projection/mod.rs b/crates/persisting-pchronicle/src/projection/mod.rs index 5a912090..a4bdeb8c 100644 --- a/crates/persisting-pchronicle/src/projection/mod.rs +++ b/crates/persisting-pchronicle/src/projection/mod.rs @@ -1,9 +1,8 @@ //! Rebuildable views derived from canonical trajectory facts. -mod agenticmd; mod storyline; -pub use agenticmd::{ +pub use crate::agenticmd::{ event_records_to_markdown_blocks, layer_stats, materialize_lance_to_markdown, materialize_markdown_path, write_markdown_projection, LayerStats, MaterializeOutcome, MaterializeStats, diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 90834249..c8195648 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -7,7 +7,6 @@ #[cfg(feature = "lance-store")] use anyhow::Context as _; -mod agenticmd_fs; #[cfg(feature = "lance-store")] mod atif_datafusion; #[cfg(feature = "lance-store")] @@ -39,7 +38,7 @@ mod storyline; #[path = "storyline/model.rs"] mod storyline_model; -pub use agenticmd_fs::{ +pub use crate::agenticmd::{ agenticmd_block_count, agenticmd_replay_json_lines, agenticmd_structural_issues, append_agenticmd_blocks, count_agenticmd_role, encode_agenticmd_block_validated, find_block_by_call_id_and_role, index_agenticmd_path, list_agenticmd_paths, diff --git a/crates/persisting-pchronicle/src/tests.rs b/crates/persisting-pchronicle/src/tests.rs index 222f455f..24b770fb 100644 --- a/crates/persisting-pchronicle/src/tests.rs +++ b/crates/persisting-pchronicle/src/tests.rs @@ -323,7 +323,7 @@ fn export_events_jsonl_debug_roundtrip_via_test_parser() { #[test] fn parse_agenticmd_document_roundtrip() { - use crate::formats::agenticmd::{ + use crate::formats::{ encode_agenticmd_document, parse_agenticmd_document, AgenticmdBlock, AgenticmdDocument, AgenticmdHeader, }; @@ -646,7 +646,7 @@ fn convert_storyline_agenticmd_preserves_dialogue_and_timing() { #[test] fn convert_agenticmd_storyline_preserves_call_id_and_seq() { use crate::convert::{agenticmd_to_storyline, storyline_to_agenticmd}; - use crate::formats::agenticmd::{AgenticmdBlock, AgenticmdDocument, AgenticmdHeader}; + use crate::formats::{AgenticmdBlock, AgenticmdDocument, AgenticmdHeader}; use serde_json::json; use std::collections::BTreeMap; From 47fb74cf357e777e974f393b3e42c507f7d6ab6b Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 20:28:02 +0800 Subject: [PATCH 14/65] ci: check pchronicle feature matrix --- .github/workflows/ci.yml | 4 +--- justfile | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa2c1747..19b31e76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,7 @@ jobs: cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check - name: Rust clippy - run: | - cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings - cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used + run: just lint-rust - name: Python lint (ruff) run: uvx ruff check persisting/ diff --git a/justfile b/justfile index cdbcd673..ceb0bb57 100644 --- a/justfile +++ b/justfile @@ -403,7 +403,7 @@ fmt-check-py: # clippy + ruff（不改写） lint: lint-rust lint-py -lint-rust: clippy-deny clippy-pchronicle-panics +lint-rust: clippy-deny clippy-pchronicle-panics clippy-pchronicle-features lint-py: uvx ruff check {{ ruff_lint_paths }} @@ -416,7 +416,12 @@ clippy-deny: cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings clippy-pchronicle-panics: - cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used + cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable + +clippy-pchronicle-features: + cargo clippy -p persisting-pchronicle --lib --no-default-features --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable + cargo clippy -p persisting-pchronicle --lib --no-default-features --features lance-store --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable + cargo clippy -p persisting-pchronicle --lib --no-default-features --features oss-store --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable # 兼容旧名 clippy: From 4c06395644f366d43be4037afb9997def236c43b Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 22:39:55 +0800 Subject: [PATCH 15/65] docs: design pchronicle document source convergence --- ...icle-document-source-convergence-design.md | 484 ++++++++++++++++++ 1 file changed, 484 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-17-pchronicle-document-source-convergence-design.md diff --git a/docs/superpowers/specs/2026-08-17-pchronicle-document-source-convergence-design.md b/docs/superpowers/specs/2026-08-17-pchronicle-document-source-convergence-design.md new file mode 100644 index 00000000..cce0b4ca --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-pchronicle-document-source-convergence-design.md @@ -0,0 +1,484 @@ +# pChronicle 文档源与查询面收敛设计 + +状态：已通过会话设计审查，待书面复核 + +日期：2026-08-17 + +范围：pChronicle 的 Storyline、Canonical Event、AgenticMD、ATIF、OpenAI Msg、ACTF、DataFusion 查询入口与公共 API + +## 1. 摘要 + +pChronicle 将 Storyline 定位为以 ATIF v1.7 语义为基准的权威轨迹模型。Storyline 三表 Lance 是该模型的二进制磁盘格式，AgenticMD 是该模型的人类可读 Markdown 编码。ATIF、OpenAI Msg 和 ACTF 都通过 Storyline 互转，不再借助隐藏的原始对象副本或进程内 provenance sidecar。 + +Canonical Event 作为第六种磁盘类型纳入统一的文档源发现和 DataFusion 打开入口，但保持独立事实语义。它通过投影产生 Storyline，不能由 Storyline 无损反向重建。 + +本设计同时完成以下清理： + +1. 消除转换层静默丢错和静默丢字段； +2. 用一个磁盘格式枚举替代包含不可能操作的 `ChronicleFormat`； +3. 让 Storyline 自身承载无损转换所需的权威信息； +4. 删除已被通用文件源取代的旧 ATIF DataSource； +5. 收敛 pChronicle 公共门面； +6. 删除 pChronicle 与 `persisting-events` 重复的轨迹协议 DTO； +7. 将现有 DataFusion 优化保留并延伸到统一的文档源能力模型。 + +## 2. 范围与非目标 + +### 2.1 本轮范围 + +- Storyline 与 ATIF 的字段对齐和无损边界； +- AgenticMD 绑定 Storyline； +- 六种磁盘格式的统一识别与读取入口； +- Canonical Event 与 Storyline 的查询面区分； +- DataFusion provider 注册、能力报告和 QueryEngine 构造入口； +- 严格转换错误； +- ATIF DataSource、重复轨迹 DTO 和过宽公共 API 的删除； +- pChronicle CLI 与 Gateway 的必要迁移。 + +### 2.2 非目标 + +- 不实施 canonical v2； +- 不修改 Canonical Event 的身份编码、Arrow schema 或 manifest 版本； +- 不改变 Canonical Event 的 append-only、writer fence 和 segment publication 语义； +- 不进入 Search、TTAS、Queue/Sampler 或 standalone dlcapt 内部实现； +- 不完成全 crate 的 typed-error 迁移； +- 不承诺把 Storyline 无损反向转换成原始 Canonical Event； +- 不为 JSON 或 Markdown 伪造 Lance scalar-index 能力。 + +## 3. 权威模型与磁盘格式 + +### 3.1 模型关系 + +```text +events.lance ──投影──► StorylineDocument + +ATIF JSON ◄──► StorylineDocument ◄──► Storyline 三表 Lance +OpenAI Msg JSON ◄──► StorylineDocument ◄──► AgenticMD Markdown +ACTF JSON ◄──► StorylineDocument +``` + +`events.lance` 是运行时事实源。Storyline 是格式转换和规范化查询域中的权威模型。两者的“权威”作用于不同边界，不互相替代。 + +### 3.2 唯一磁盘格式枚举 + +```rust +pub enum DocumentFormat { + CanonicalEvent, + Storyline, + AgenticMd, + Atif, + OpenaiMsg, + Actf, +} +``` + +各 variant 的精确定义： + +| Variant | 磁盘表示 | 逻辑查询表 | +|---|---|---| +| `CanonicalEvent` | `events.lance` manifest 与 segments | `events` | +| `Storyline` | `runs`、`steps`、`tool_calls`、`objects` Lance | `runs`、`steps`、`tool_calls` | +| `AgenticMd` | Storyline Markdown 文件 | `runs`、`steps`、`tool_calls` | +| `Atif` | ATIF JSON、JSONL 或 NDJSON | `runs`、`steps`、`tool_calls` | +| `OpenaiMsg` | OpenAI message JSON | `runs`、`steps`、`tool_calls` | +| `Actf` | ACTF JSON | `runs`、`steps`、`tool_calls` | + +`DocumentFormat` 只描述磁盘表示，不承诺每个 variant 支持相同的写入输入类型。 + +## 4. Storyline 与 ATIF 对齐 + +### 4.1 正式字段原则 + +Storyline 的正式字段以 ATIF Trajectory、Step、ToolCall 和 Observation 为基准： + +- `StorylineDocument` 对应 ATIF Trajectory； +- `StorylineTurn` 对应 ATIF Step； +- `StorylineToolCall` 对应 ATIF ToolCall； +- `agent`、`notes`、`final_metrics`、`continued_trajectory_ref`、`observation`、`metrics` 与 ATIF 保持同义。 + +Storyline 只保留以下已接受增量： + +- required `agent.id`； +- parent/children 外链； +- `kind`； +- `latency_ms` 与 `ttft_ms`； +- tool call `duration_ms`； +- pChronicle 现有存储所需的 run/session 对应关系。 + +### 4.2 必须补齐的 ATIF 语义 + +- `StorylineToolCall` 增加 ATIF inline `result`； +- 对 ATIF 要求保真的值使用能区分“字段缺失”“显式 null”“实际值”的表示； +- ATIF observation 解析失败必须返回错误，不能删除 observation； +- ATIF schema/version 信息必须能经过 Storyline 三表 split/reconstruct 保留； +- ATIF adapter 必须是近似结构映射，不保存 `_pchronicle_atif_tool_call` 等原始对象副本。 + +`AtifTrajectory`、`AtifStep` 等可以保留为私有 wire DTO，但不再与 Storyline 一起构成两套公开领域模型。 + +## 5. 无损往返与 residual extensions + +### 5.1 权威性规则 + +Storyline 自身必须包含目标转换器所需的全部信息。禁止使用以下机制： + +- 进程内 provenance sidecar； +- 绑定单一来源格式的 provenance enum； +- `_pchronicle_*` 私有键； +- 完整原始 document、record、step 或 tool-call 对象副本。 + +### 5.2 residual 规则 + +不能映射到 ATIF/Storyline 正式字段的外围格式字段，存入对应语义层级的 `extra`： + +- ACTF 文档级剩余字段进入 document `extra`； +- ACTF attempt/step 剩余字段进入对应 Storyline/turn `extra`； +- tool 与 observation 的事件专属字段进入对应对象扩展； +- OpenAI 文件容器、相对路径等文件级信息进入 document `extra`； +- OpenAI ordinal 与未映射 record 字段进入 turn `extra`。 + +每个 adapter 必须先从源对象移除已经映射的键，再保存 residual。导出时先根据 Storyline 正式字段生成目标对象，再合并无冲突 residual。若 residual 与正式字段映射到同一目标键，正式字段获胜。 + +### 5.3 保真边界 + +以下路径保证 JSON 数据模型级无损： + +```text +ATIF → Storyline Lance → ATIF +ACTF → Storyline Lance → ACTF +OpenAI Msg → Storyline Lance → OpenAI Msg +``` + +保留键值、显式 null、未知字段、数组顺序、ACTF attempt 分组和 OpenAI 多 session 关系。不保证空白、缩进和对象键顺序逐字节一致。 + +跨格式 `F1 → Storyline → F2` 输出 F2 能表达的全部语义。如果 F2 没有相应字段或扩展通道，不承诺再经过 F2 恢复 F1 私有字段。 + +## 6. AgenticMD 作为 Storyline Markdown + +AgenticMD 不再定义独立领域模型。公开语义接口为： + +```rust +pub fn parse_agenticmd(input: &str) -> Result; +pub fn encode_agenticmd(story: &StorylineDocument) -> Result; +``` + +约束： + +- frontmatter 映射 `StorylineDocument`； +- 每个正文 block 映射 `StorylineTurn`； +- tool calls、observation、metrics、模型和时延都使用 Storyline 字段； +- 私有 Markdown AST 只负责语法、byte span 和增量编辑； +- 结构化 HTML comment 只能序列化 Storyline 类型； +- 删除 AgenticMD correlation keys 在 `turn.extra` 中的搬运； +- 删除公开 `AgenticmdDocument`、`AgenticmdBlock`、`AgenticmdHeader`。 + +Gateway 的投影路径统一为： + +```text +EventRecord[] → Storyline projection → AgenticMD encoding +``` + +## 7. 文档源读取与类型化写入 + +### 7.1 统一读取入口 + +```rust +pub async fn open_document( + format: DocumentFormat, + path: &Path, +) -> Result; +``` + +`DocumentSource` 是公开 struct，其具体 provider enum 保持私有。默认入口使用各 provider +已有的安全默认值；确有高级配置需求的调用者继续使用对应的类型化 provider 构造器，本轮 +不设计一个包含大量互斥字段的通用 options 对象。 + +`DocumentSource` 提供： + +```rust +impl DocumentSource { + pub fn format(&self) -> DocumentFormat; + pub async fn project_storylines(&self) -> Result>; + pub fn register_datafusion( + &self, + context: &SessionContext, + ) -> Result; +} +``` + +六种磁盘源都能打开、查询并投影为 Storyline。 + +### 7.2 类型化写入 + +写入不使用一个接受所有 enum variant 的通用 `save`： + +```rust +RawEventLanceStore::append_events(&[EventRecord]); +StorylineLanceStore::replace_storylines(&[StorylineDocument]); +encode_agenticmd(&StorylineDocument); +write_atif(&[StorylineDocument]); +write_openai_msg(&[StorylineDocument]); +write_actf(&[StorylineDocument]); +``` + +禁止 `save(DocumentFormat::CanonicalEvent, &[StorylineDocument])`。Storyline 到 EventRecord 的现有合成转换只能保留为明确命名的调试/导出能力，不能写入或宣称重建 canonical facts。 + +`storyline.json` 不再是一等 `DocumentFormat`。`StorylineDocument` 可以继续使用 serde JSON，供测试、调试和内部传输使用。 + +## 8. DataFusion 查询设计 + +### 8.1 统一内部接口 + +```rust +pub(crate) trait QueryDocumentSource { + fn format(&self) -> DocumentFormat; + fn tables(&self) -> QueryTables; + fn capabilities(&self) -> QueryCapabilities; + fn register(&self, context: &SessionContext) -> Result<()>; +} +``` + +```rust +pub struct QueryCapabilities { + pub projection_pushdown: bool, + pub filter_pushdown: FilterPushdown, + pub limit_pushdown: bool, + pub scalar_indexes: bool, + pub streaming_decode: bool, + pub late_content_materialization: bool, + pub snapshot_consistent: bool, +} +``` + +```rust +pub enum QueryTables { + Events, + Storyline, +} + +pub enum FilterPushdown { + Unsupported, + Inexact, + Exact, + ExpressionDependent, +} +``` + +`ExpressionDependent` 表示 provider 必须按具体表达式调用 +`supports_filters_pushdown` 决定 `Unsupported`、`Inexact` 或 `Exact`，不能把混合能力压成 +一个过度承诺的静态值。 + +能力必须按 provider 的真实实现报告，不能为了接口统一虚报 `Exact` filter pushdown。 + +### 8.2 各格式能力 + +#### Canonical Event + +- 注册 `events` 表； +- 保留 Lance projection、exact filter pushdown、scalar index、segment union、pinned manifest、append-order range scan； +- 默认不实时注册 `runs/steps/tool_calls`； +- 需要 Storyline 查询面时，优先使用 lineage 新鲜的 Storyline Lance 投影； +- 无投影时只能在已有 row/byte budget 内执行 bounded fallback。 + +#### Storyline Lance + +- 注册 `runs/steps/tool_calls`； +- 保留 Lance projection、非内容列 exact filter、scalar index、条件式 limit pushdown； +- 保留 content sidecar late hydration 与 preview 模式； +- 内容列谓词继续按当前安全规则报告 unsupported 或 fail closed。 + +#### ATIF + +- 注册 `runs/steps/tool_calls`； +- 保留文件裁剪、step filter inexact pushdown、字段投影、流式解析、bounded-memory reader 和 cache； +- 不退化现有 ATIF streaming metrics 与测试。 + +#### OpenAI Msg 与 ACTF + +- 注册 `runs/steps/tool_calls`； +- 保留文件级裁剪、解析缓存、投影后的 Arrow batch、文件大小和并发限制； +- 未实现行级安全下推前报告 unsupported。 + +#### AgenticMD + +- 解析为 Storyline 后复用 Storyline Arrow row codec； +- 注册 `runs/steps/tool_calls`； +- 第一版仅报告文件级裁剪与 projection； +- 行级 filter pushdown 初始为 unsupported； +- 不为 DataFusion 创建独立 AgenticMD schema。 + +### 8.3 QueryEngine 收敛 + +QueryEngine 公开构造入口收敛为： + +```rust +pub async fn ChronicleQueryEngine::open( + format: DocumentFormat, + path: impl AsRef, + options: ChronicleQueryExecutionOptions, +) -> Result; +``` + +删除格式专属的重复 `open_*`、`from_*` 构造器。后端信息改为统一结构： + +```rust +pub struct QueryBackendInfo { + pub format: DocumentFormat, + pub tables: QueryTables, + pub capabilities: QueryCapabilities, + pub source_count: usize, + pub snapshot: Option, +} +``` + +```rust +pub enum QuerySnapshot { + CanonicalEvent { + fact_version: u64, + fact_rows: u64, + layout_revision: u64, + }, + Storyline { + generation: String, + }, +} +``` + +文本文件源没有独立事务快照，`snapshot` 为 `None`；Catalog 自己保留跨数据集 snapshot +标识，不伪装成单一文档源快照。 + +统一 SessionContext、memory/spill 配置、SQL 校验、metrics 和 catalog 调度；provider 内部优化保持分层实现。 + +## 9. 严格错误语义 + +转换层必须删除： + +- `messages_value().ok()`； +- `response_value().ok()`； +- AgenticMD YAML 错误后的 `unwrap_or_default()`； +- ATIF observation 解析错误后的 `.ok()`； +- 对需要耐久保证的写入忽略 `sync_all` 结果。 + +新增有限的转换错误： + +```rust +Error::InvalidDocument { + format: DocumentFormat, + path: Option, + location: Option, + message: String, +} + +Error::UnsupportedCardinality { + format: DocumentFormat, + stories: usize, +} +``` + +错误必须尽可能指出文件、document、record、step 或字段位置。本轮不顺带替换所有 `anyhow` 或 `Error::Other`。 + +## 10. 删除旧 ATIF DataSource + +删除： + +- `AtifDataSource`； +- `AtifDataSourceOptions`； +- `ChronicleQueryEngine::from_atif_source`； +- 专属 provider 注册、统计和仅覆盖旧入口的测试。 + +保留通用文件 provider、ATIF streaming parser、`AtifReader` 和加载辅助能力。内存查询测试使用临时 ATIF 文件或通用文件源。 + +## 11. 删除重复轨迹协议 DTO + +pChronicle 删除全部 `Trajectory*Request/Response` 以及 `operations::trajectory` wire adapter。控制协议只由 `persisting-events` 定义。 + +CLI append 数据流为： + +```text +persisting_events::TrajectoryAppendRequest + → StoryCoords + → pChronicle storage API + → persisting_events::TrajectoryAppendResponse +``` + +禁止 serde JSON request/response transcode。Replay、stats、materialize 和 extract 如仍有调用者，保留领域函数与领域结果，但不复制 wire DTO。 + +## 12. 公共 API + +公开架构收敛为： + +```text +persisting_pchronicle::model +persisting_pchronicle::document +persisting_pchronicle::storage +persisting_pchronicle::query +``` + +- `model`：Storyline 与 EventRecord 相关权威类型； +- `document`：`DocumentFormat`、`DocumentSource`、打开和格式写出接口； +- `storage`：主要 store、坐标、结果和配置； +- `query`：QueryEngine、query options、backend info 和 capabilities。 + +Arrow row codec、Markdown AST、格式 parser、provider、lock、manifest 实现、内部常量和 projection helper 全部为 private 或 `pub(crate)`。不保留 deprecated 旧路径；当前 workspace 消费者一次迁移。 + +## 13. 实施顺序 + +### 阶段一：严格转换与 ATIF 对齐 + +- 先增加畸形 OpenAI、AgenticMD YAML 和 ATIF observation 的失败测试； +- 补齐 ATIF 对齐字段与缺失/null 表达； +- 删除 `_pchronicle_atif`； +- 将 ACTF/OpenAI 原始副本改成分层 residual； +- 验证三种外部格式经过 Storyline serde 和 Lance 的无损往返。 + +### 阶段二：AgenticMD 绑定 Storyline + +- 先增加 Storyline 与 AgenticMD 完整相等测试； +- 私有化 Markdown AST； +- 删除独立 AgenticMD 领域类型； +- 迁移 Gateway 投影。 + +### 阶段三：统一文档源与 DataFusion + +- 引入六 variant `DocumentFormat`； +- 实现 `DocumentSource` 与 query capabilities； +- 将现有 Event、Storyline 和 file providers 接入； +- 增加 AgenticMD provider； +- 收敛 QueryEngine 构造器； +- 保留每种 provider 的现有优化和真实性声明。 + +### 阶段四：删除重复入口和收敛门面 + +- 删除旧 ATIF DataSource； +- 删除重复轨迹 DTO 与 serde transcode； +- 迁移 CLI 和 workspace 消费者； +- 私有化实现模块并修复 rustdoc。 + +## 14. 测试与验收 + +### 14.1 必须覆盖的行为 + +1. 畸形 OpenAI JSON、AgenticMD YAML、ATIF observation 返回错误； +2. `Storyline → AgenticMD → Storyline` 完整结构相等； +3. `ATIF → Storyline Lance → ATIF` 数据模型相等； +4. `ACTF → Storyline Lance → ACTF` 保留 null、未知字段、数组顺序和 attempt 分组； +5. `OpenAI → Storyline Lance → OpenAI` 保留文件容器、多 session、ordinal 和未知字段； +6. Canonical Event 只注册 `events`，且保留 projection/filter/index/snapshot 行为； +7. Storyline Lance 保留 content late hydration 和 preview 行为； +8. ATIF 保留 streaming projection、filter pruning 和 bounded-memory 指标； +9. OpenAI/ACTF 不虚报 exact row filter pushdown； +10. AgenticMD provider 复用 Storyline 三表 schema； +11. CLI append 不再执行 serde transcode； +12. rustdoc 不再平铺或链接内部实现细节。 + +### 14.2 最终验证命令 + +```bash +cargo test -p persisting-pchronicle --no-default-features --locked +cargo test -p persisting-pchronicle --features lance-store --locked +cargo test -p persisting-pchronicle-cli --locked +cargo test -p persisting-gateway --locked +cargo clippy -p persisting-pchronicle --all-targets --features lance-store --locked -- -D warnings +RUSTDOCFLAGS="-D warnings" cargo doc -p persisting-pchronicle --no-deps --locked +``` + +Search、TTAS、Queue/Sampler 和 standalone dlcapt 的失败不属于本轮验收标准。 From c13486cc1a32a6246ac6527595d1e9729fb9aad0 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:01:12 +0800 Subject: [PATCH 16/65] docs: plan pchronicle document source convergence --- ...-pchronicle-document-source-convergence.md | 435 ++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-17-pchronicle-document-source-convergence.md diff --git a/docs/superpowers/plans/2026-08-17-pchronicle-document-source-convergence.md b/docs/superpowers/plans/2026-08-17-pchronicle-document-source-convergence.md new file mode 100644 index 00000000..34f1eea7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-pchronicle-document-source-convergence.md @@ -0,0 +1,435 @@ +# pChronicle 文档源与查询面收敛实施计划 + +> **供 agentic worker 使用：** 必须使用 `superpowers:executing-plans` 逐项执行；每个行为改动遵循 `superpowers:test-driven-development`，完成声明前使用 `superpowers:verification-before-completion`。 + +**目标：** 以 ATIF 对齐的 Storyline 为权威转换模型，消除转换层静默丢数据，统一六种磁盘文档格式及其 DataFusion 打开入口，并删除旧 ATIF DataSource、重复轨迹 DTO 和过宽公共门面。 + +**架构：** Canonical Event 继续作为 append-only 运行时事实源，只能投影为 Storyline；Storyline 三表 Lance 是权威轨迹模型的二进制表示；AgenticMD 是 Storyline 的 Markdown 编码；ATIF、ACTF 与 OpenAI Msg 通过 Storyline 的正式字段和分层 residual extensions 实现 JSON 数据模型级无损往返。读取使用统一 `DocumentSource`，写入保持类型化，DataFusion provider 通过能力描述共享入口但不虚报优化。 + +**技术栈：** Rust 2021、Serde/serde_json/serde_yaml、Arrow/Lance、DataFusion、Tokio、Cargo。 + +## 全局约束 + +- 不实施 canonical v2，不改变 Canonical Event schema、manifest、append、fence 或 segment publication 语义。 +- 不进入 Search、TTAS、Queue/Sampler 或 standalone `persisting-dlcapt`。 +- 不修改、删除或提交用户已有未跟踪文件。 +- 不保留 deprecated 旧 API；当前 workspace 消费者同批迁移。 +- 无损指 JSON 数据模型级相等，不要求空白、缩进或对象键顺序逐字节一致。 +- 每个任务都先新增一个会因当前缺陷失败的测试并实际观察 RED；未观察到预期失败时停止并修正测试。 +- 每个提交只包含本任务文件；提交前运行对应测试与 `git diff --check`。 + +--- + +### 任务 1：引入唯一磁盘格式枚举与严格文档错误 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/format.rs` +- 修改：`crates/persisting-pchronicle/src/error.rs` +- 修改：`crates/persisting-pchronicle/src/tests.rs` + +**接口：** + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DocumentFormat { + CanonicalEvent, + Storyline, + AgenticMd, + Atif, + OpenaiMsg, + Actf, +} + +Error::InvalidDocument { + format: DocumentFormat, + path: Option, + location: Option, + message: String, +} + +Error::UnsupportedCardinality { + format: DocumentFormat, + stories: usize, +} + +Error::SourceBudgetExceeded { + format: DocumentFormat, + path: Option, + budget: String, +} +``` + +- [ ] 增加枚举解析/显示测试，要求只接受规范名称 `canonical-event`、`storyline`、`agenticmd`、`atif`、`openai-msg`、`actf`，并验证错误包含 format/location。 +- [ ] 运行 `cargo test -p persisting-pchronicle --no-default-features document_format --locked`，确认因类型不存在而 RED。 +- [ ] 实现 `DocumentFormat`、`Display`、`FromStr` 与三个错误 variant；本任务暂不删除 `ChronicleFormat`，以便后续按消费者分批迁移。 +- [ ] 重跑目标测试并运行 `cargo check -p persisting-pchronicle --no-default-features --locked`。 +- [ ] 提交：`refactor: define pchronicle document formats`。 + +### 任务 2：封堵转换层静默失败 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/convert/openai_msg.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/codec.rs` +- 修改：`crates/persisting-pchronicle/src/convert/atif.rs` +- 修改：`crates/persisting-pchronicle/src/store/events/manifest.rs` +- 修改：`crates/persisting-pchronicle/src/store/attempt_registry.rs` +- 修改：`crates/persisting-pchronicle/src/tests.rs` +- 修改：`crates/persisting-pchronicle/src/store/events/tests.rs` + +- [ ] 增加三类 RED 测试：OpenAI record 的 `messages`/`response` 类型错误必须定位 record；AgenticMD frontmatter YAML 类型错误必须返回 `InvalidDocument`；ATIF observation 结构错误必须定位 step/observation。 +- [ ] 增加目录 durability 错误的可注入单元测试：将目录同步抽为接收 `&File` 的私有 helper，helper 必须传播 `sync_all` 错误；不得依赖平台权限碰运气。 +- [ ] 分别运行： + +```bash +cargo test -p persisting-pchronicle --no-default-features malformed_openai --locked +cargo test -p persisting-pchronicle --no-default-features malformed_agenticmd_yaml --locked +cargo test -p persisting-pchronicle --no-default-features malformed_atif_observation --locked +``` + + 确认当前 `.ok()`、`unwrap_or_default()` 或忽略错误使断言失败。 +- [ ] 用 `?` 和带 path/record/step location 的 `InvalidDocument` 替换静默降级；只修正持久化路径中的 `let _ = directory.sync_all()`，不机械替换语义正确的 `Option::unwrap_or_default()`。 +- [ ] 重跑上述测试，再运行 `cargo test -p persisting-pchronicle --no-default-features --locked`。 +- [ ] 提交：`fix: reject malformed pchronicle documents`。 + +### 任务 3：让 Storyline 完整表达 ATIF 字段存在性 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/formats/storyline.rs` +- 修改：`crates/persisting-pchronicle/src/atif.rs` +- 修改：`crates/persisting-pchronicle/src/convert/atif.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/model.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/rows.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/mod.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/tests.rs` +- 修改：`crates/persisting-pchronicle/tests/atif_lance_corpus.rs` + +**模型：** + +```rust +#[derive(Debug, Clone, Default, PartialEq)] +pub enum FieldPresence { + #[default] + Missing, + Null, + Value(T), +} + +pub struct StorylineDocument { + pub schema_version: Option, + pub attempt_id: Option, + // existing fields +} + +pub struct StorylineToolCall { + // existing fields + pub result: FieldPresence, +} +``` + +- [ ] 增加 serde RED 测试，分别输入 tool call result 缺失、显式 `null`、实际 JSON 值，要求三态序列化后仍可区分。 +- [ ] 增加 split/reconstruct RED 测试，要求 `schema_version`、可选 `attempt_id` 与三种 result 状态经过 `StorylineTables` 完整相等；外部格式导入时 `attempt_id` 必须为 `None`。 +- [ ] 实现 `FieldPresence` 的 serde/default/skip helper；将 ATIF wire DTO 的 inline result 迁为相同存在性语义。 +- [ ] 在 StoryRunRow/StoryToolCallRow 与 Arrow schema/codec 中增加列；旧数据缺列时使用 `Missing`/`None`，不得把显式 null 当缺失。 +- [ ] 更新 ATIF 结构映射，删除 `_pchronicle_atif*` 原始对象副本，只映射正式字段与真正未知的 `extra`。 +- [ ] 运行： + +```bash +cargo test -p persisting-pchronicle --no-default-features field_presence --locked +cargo test -p persisting-pchronicle --features lance-store storyline --locked +cargo test -p persisting-pchronicle --features lance-store --test atif_lance_corpus --locked +``` + +- [ ] 提交：`feat: align storyline with atif presence semantics`。 + +### 任务 4：用分层 residual 实现 ACTF 权威往返 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/convert/actf.rs` +- 修改：`crates/persisting-pchronicle/src/formats/actf.rs` +- 修改：`crates/persisting-pchronicle/src/tests.rs` +- 修改：`crates/persisting-pchronicle/tests/import_roundtrip_fixtures.rs` + +**扩展键：** `persisting.dev/actf/v1`。该值只保存已移除正式映射键后的 document/attempt/step/tool/observation residual，不保存完整原对象。 + +- [ ] 增加 RED 测试：导入带未知字段、显式 null、多个 attempt 和有序数组的 ACTF；修改 Storyline 正式 message/reasoning 后导出，要求修改生效且 residual 完整保留。 +- [ ] 增加断言：所有层级 `extra` 中都不存在完整源 step/trajectory，也不存在 `_pchronicle_` 键。 +- [ ] 运行 `cargo test -p persisting-pchronicle --no-default-features actf_residual --locked`，确认当前 raw-step 回放使 Storyline 修改被覆盖。 +- [ ] 实现“先移除 mapped keys，再保存 residual；导出先生成正式字段，再合并无冲突 residual”的双向映射。冲突时正式字段获胜，并通过 `tracing::warn!` 输出包含来源格式、源键和目标键的结构化诊断。 +- [ ] 运行目标测试与 `cargo test -p persisting-pchronicle --test import_roundtrip_fixtures --features lance-store --locked`。 +- [ ] 提交：`fix: make actf residual roundtrips authoritative`。 + +### 任务 5：用分层 residual 实现 OpenAI Msg 权威往返 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/convert/openai_msg.rs` +- 修改：`crates/persisting-pchronicle/src/formats/openai_corpus.rs` +- 修改：`crates/persisting-pchronicle/src/tests.rs` +- 修改：`crates/persisting-pchronicle/tests/import_roundtrip_fixtures.rs` + +**扩展键：** `persisting.dev/openai-msg/v1`。document residual 保存容器、相对路径和未映射 envelope；turn residual 保存 ordinal 与删除正式键后的 record residual。 + +- [ ] 增加 RED 测试：多 session、多 record、未知字段、显式 null 与有序数组导入后，修改 Storyline user/assistant 内容再导出；要求正式修改生效，文件分组/ordinal/residual 不变。 +- [ ] 增加断言：`extra` 不含完整 raw record，不含 `_pchronicle_` 键。 +- [ ] 运行 `cargo test -p persisting-pchronicle --no-default-features openai_residual --locked`，确认当前 raw record 优先导致 RED。 +- [ ] 实现分层 residual 合并及 stable ordering；正式 Storyline 字段最后写入目标 map。发生键冲突时输出与 ACTF 相同字段结构的 `tracing::warn!` 诊断。 +- [ ] 重跑目标测试与 import fixture 测试。 +- [ ] 提交：`fix: make openai residual roundtrips authoritative`。 + +### 任务 6：建立三种格式经 Storyline Lance 的无损验收 + +**文件：** +- 新建：`crates/persisting-pchronicle/tests/storyline_lance_roundtrip.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/mutation.rs`（仅测试暴露的缺陷需要时） +- 修改：`crates/persisting-pchronicle/src/store/storyline/rows.rs`（仅测试暴露的缺陷需要时） + +- [ ] 用现有 ATIF/ACTF/OpenAI fixture 构造共享 helper：解析源 JSON Value → Storyline → 临时 Lance replace/load → 导出 Value → 语义比较。 +- [ ] 先运行 `cargo test -p persisting-pchronicle --features lance-store --test storyline_lance_roundtrip --locked`，确认至少因未保存的新字段/residual 而 RED。 +- [ ] 只修复 Lance split/reconstruct 或 codec 暴露的丢失，不在本任务改变 adapter 语义。 +- [ ] 要求三条路径均保留 null、未知字段、数组顺序；ACTF 保留 attempt 分组，OpenAI 保留多 session 与 ordinal。 +- [ ] 重跑测试，提交：`test: enforce lossless storyline lance roundtrips`。 + +### 任务 7：将 AgenticMD 语义接口绑定到 Storyline + +**文件：** +- 修改：`crates/persisting-pchronicle/src/agenticmd/mod.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/codec.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/convert.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/frontmatter.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/body.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/mapping/*.rs` +- 修改：`crates/persisting-pchronicle/src/lib.rs` +- 修改：`crates/persisting-pchronicle/src/tests.rs` + +**公开接口：** + +```rust +pub fn parse_agenticmd(input: &str) -> Result; +pub fn encode_agenticmd(story: &StorylineDocument) -> Result; +``` + +- [ ] 增加 RED 测试：包含 document、turn、tool call/result、observation、metrics、模型、latency、unknown extra 的 Storyline，执行 encode/parse 后完整结构相等。 +- [ ] 将 `AgenticmdDocument`、`AgenticmdBlock`、`AgenticmdHeader` 降为私有 Markdown syntax AST；AST 只能在 `agenticmd/` 内出现。 +- [ ] 由 frontmatter/body/comment 直接映射 Storyline 字段，删除 correlation keys 在 `turn.extra` 中的搬运。 +- [ ] 保留必要的私有 byte-span/incremental edit helper，但公共函数参数和返回值不得暴露 AST。 +- [ ] 运行： + +```bash +cargo test -p persisting-pchronicle --no-default-features agenticmd_storyline --locked +cargo check -p persisting-pchronicle --no-default-features --locked +``` + +- [ ] 提交：`refactor: bind agenticmd encoding to storyline`。 + +### 任务 8：迁移 Gateway 到 EventRecord → Storyline → AgenticMD + +**文件：** +- 修改：`crates/persisting-pchronicle/src/agenticmd/projection.rs` +- 修改：`crates/persisting-pchronicle/src/agenticmd/fs.rs` +- 修改：`crates/persisting-gateway/src/projection/markdown.rs` +- 修改：`crates/persisting-gateway/src/projection/pipeline.rs` +- 修改：`crates/persisting-gateway/src/projection/reconcile.rs` +- 修改：`crates/persisting-gateway/src/projection/frontmatter.rs` +- 修改：`crates/persisting-gateway/src/projection/dialogue/block.rs` +- 修改：`crates/persisting-gateway/src/projection/dialogue/draft.rs` +- 修改：`crates/persisting-gateway/src/projection/dialogue/tests.rs` +- 修改：`crates/persisting-gateway/tests/agenticmd_bridge.rs` +- 修改：`crates/persisting-gateway/tests/agenticmd_golden.rs` + +**边界：** Gateway 只创建/修改 `StorylineDocument` 与 `StorylineTurn`；pChronicle 内部决定 Markdown block、frontmatter 和增量文件编辑。 + +- [ ] 先将 golden/bridge 断言改为通过 `parse_agenticmd` 检查 Storyline 语义，并运行测试确认公共 AST 仍被依赖而 RED。 +- [ ] 为增量写入提供 Storyline 高层接口，例如 `upsert_agenticmd_turn(path, document_meta, turn)`；公开参数不得包含 AgenticMD AST。 +- [ ] 将稳定事件投影为 Storyline turn；draft 同样使用临时 Storyline turn，完成时由同一 call id 覆盖。保持 skip/dedup/atomic rewrite 行为。 +- [ ] 删除 Gateway 对 `Agenticmd*` 类型、frontmatter formatter 和 block builder 的导入。 +- [ ] 运行： + +```bash +cargo test -p persisting-gateway --lib projection:: --locked +cargo test -p persisting-gateway --test agenticmd_bridge --locked +cargo test -p persisting-gateway --test agenticmd_golden --locked +``` + +- [ ] 提交：`refactor: project gateway markdown through storyline`。 + +### 任务 9：实现统一 DocumentSource 和真实能力模型 + +**文件：** +- 新建：`crates/persisting-pchronicle/src/document.rs` +- 新建：`crates/persisting-pchronicle/src/store/document_source.rs` +- 新建：`crates/persisting-pchronicle/src/store/agenticmd_datafusion.rs` +- 修改：`crates/persisting-pchronicle/src/store/files/mod.rs` +- 修改：`crates/persisting-pchronicle/src/store/events/datafusion.rs` +- 修改：`crates/persisting-pchronicle/src/store/storyline/datafusion.rs` +- 修改：`crates/persisting-pchronicle/src/store/mod.rs` +- 修改：`crates/persisting-pchronicle/src/lib.rs` +- 新建：`crates/persisting-pchronicle/tests/document_source.rs` + +**接口：** 按批准规格实现 `open_document`、`DocumentSource::{format,project_storylines,for_each_storyline,register_datafusion}`、`QueryTables`、`FilterPushdown`、`QueryCapabilities` 与私有 `QueryDocumentSource`。 + +- [ ] 增加六种格式打开测试及 provider 能力矩阵测试；Canonical Event 只注册 `events`，其他五种注册 `runs/steps/tool_calls`。 +- [ ] 增加 AgenticMD provider RED 测试，要求它使用 Storyline 三表 Arrow schema，filter pushdown 为 Unsupported，projection 可用。 +- [ ] 增加预算 RED 测试：`project_storylines` 超过行/字节预算必须返回 `SourceBudgetExceeded`；`for_each_storyline` 在同一输入上保持有界并完整遍历，不得静默截断。 +- [ ] 实现私有 provider enum，复用已有 Event/Storyline/file provider；不得把一个包含互斥字段的 options struct 暴露为统一 API。 +- [ ] 确保能力真值：Event 保留 exact/index/snapshot；Storyline 保留 late hydration；ATIF 为 inexact/streaming；OpenAI/ACTF 行 filter unsupported；AgenticMD 行 filter unsupported。 +- [ ] 运行： + +```bash +cargo test -p persisting-pchronicle --features lance-store --test document_source --locked +cargo test -p persisting-pchronicle --features lance-store --test direct_file_query --locked +``` + +- [ ] 提交：`feat: unify pchronicle document sources`。 + +### 任务 10：收敛 QueryEngine 并保持 DataFusion 优化 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/store/query_engine.rs` +- 修改：`crates/persisting-pchronicle/src/store/local_query_manifest.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/provider.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/source.rs` +- 修改：`crates/persisting-pchronicle/tests/query_engine.rs` +- 修改：`crates/persisting-pchronicle/tests/direct_file_query.rs` + +**公开接口：** + +```rust +pub async fn ChronicleQueryEngine::open( + format: DocumentFormat, + path: impl AsRef, + options: ChronicleQueryExecutionOptions, +) -> Result; +``` + +- [ ] 将 tests 迁到唯一 `open`，先增加 `QueryBackendInfo`/`QuerySnapshot`/capabilities 精确断言并确认现有格式专用 backend enum 无法满足；Canonical Event snapshot 必须报告 `format_version`，既有无版本 manifest 为 `1`。 +- [ ] 用 `DocumentSource` 统一 SessionContext、memory/spill、SQL validation 与 metrics；provider 注册仍调用原有优化实现。 +- [ ] 删除公开 `open_*`、`from_*` 构造器和 `ChronicleQueryBackend`，改用批准规格中的 `QueryBackendInfo`。 +- [ ] 回归 exact filter/scalar index/pinned manifest、Storyline preview/late hydration、ATIF streaming projection/bounded metrics。 +- [ ] 运行： + +```bash +cargo test -p persisting-pchronicle --features lance-store --test query_engine --locked +cargo test -p persisting-pchronicle --features lance-store --test direct_file_query --locked +cargo test -p persisting-pchronicle --features lance-store --test production_scale --locked +``` + +- [ ] 提交：`refactor: converge pchronicle query engine entrypoints`。 + +### 任务 11：删除旧 ATIF DataSource + +**文件：** +- 删除：`crates/persisting-pchronicle/src/store/atif_datafusion.rs` +- 修改：`crates/persisting-pchronicle/src/store/mod.rs` +- 修改：`crates/persisting-pchronicle/src/lib.rs` +- 修改：`crates/persisting-pchronicle/tests/query_engine.rs` +- 修改：`crates/persisting-pchronicle/benches/*.rs`（仅引用旧源的 benchmark） + +- [ ] 先把旧源测试改为临时 ATIF 文件 + `DocumentSource`/通用 `FileTrajectoryDataSource`，保持 invalid input、duplicate step、batch size、file count 和 filter 行为覆盖。 +- [ ] 删除 `AtifDataSource`、`AtifDataSourceOptions`、`from_atif_source` 及专属 re-export；保留 `AtifReader`、stream parser 和通用文件源。 +- [ ] 运行 `rg -n "AtifDataSource|from_atif_source" crates/persisting-pchronicle --glob '!target/**'`，预期无输出。 +- [ ] 运行 pChronicle query/direct-file tests 和 `cargo check -p persisting-pchronicle --all-targets --features lance-store --locked`。 +- [ ] 提交：`refactor: remove legacy atif data source`。 + +### 任务 12：删除重复轨迹协议 DTO 与 serde transcode + +**文件：** +- 修改：`crates/persisting-pchronicle/src/messages.rs` +- 删除：`crates/persisting-pchronicle/src/operations/trajectory/mod.rs` +- 删除：`crates/persisting-pchronicle/src/operations/trajectory/tests.rs` +- 修改：`crates/persisting-pchronicle/src/operations/mod.rs` +- 修改：`crates/persisting-pchronicle/src/operations/dispatch.rs` +- 修改：`crates/persisting-pchronicle/src/operations/bridge.rs` +- 修改：`crates/persisting-pchronicle-cli/src/control.rs` +- 修改：`crates/persisting-pchronicle-cli/src/tests.rs` + +- [ ] 增加 CLI append 测试，直接构造 `persisting_events::TrajectoryAppendRequest` 并断言返回 `persisting_events::TrajectoryAppendResponse`；以类型检查保证无 serde 中转。 +- [ ] 将 append/replay/stats/materialize/extract 的存储调用提取为领域函数，参数使用 `StoryCoords`、EventRecord 和领域 options/result，不复制 wire DTO。 +- [ ] CLI control 对 persisting-events request 做显式字段映射，直接构建同 crate response；删除 `transcode` helper。 +- [ ] 删除 pChronicle `Trajectory*Request/Response` 与 wire adapter，只保留确有调用者的领域结果类型。 +- [ ] 运行： + +```bash +cargo test -p persisting-pchronicle-cli --locked +cargo test -p persisting-pchronicle operations --features lance-store --locked +rg -n "struct Trajectory.*(Request|Response)|fn transcode" crates/persisting-pchronicle crates/persisting-pchronicle-cli +``` + + 最后一条不得命中 pChronicle 自定义 wire DTO 或 transcode。 +- [ ] 提交：`refactor: use persisting events trajectory protocol directly`。 + +### 任务 13：完全替换 ChronicleFormat 并收紧公共门面 + +**文件：** +- 修改：`crates/persisting-pchronicle/src/format.rs` +- 修改：`crates/persisting-pchronicle/src/convert/mod.rs` +- 修改：`crates/persisting-pchronicle/src/formats/detect.rs` +- 修改：`crates/persisting-pchronicle/src/lib.rs` +- 新建：`crates/persisting-pchronicle/src/model.rs` +- 新建：`crates/persisting-pchronicle/src/storage.rs` +- 新建：`crates/persisting-pchronicle/src/query.rs` +- 修改：`crates/persisting-pchronicle-cli/src/exchange.rs` +- 修改：`crates/persisting-pchronicle-cli/src/lib.rs` +- 修改：`crates/persisting-pchronicle/benches/lance_vs_json.rs` +- 修改：`crates/persisting-pchronicle/benches/pchronicle_criterion.rs` +- 修改：`crates/persisting-pchronicle/benches/projection_pipeline.rs` +- 修改：`crates/persisting-pchronicle/src/formats/mod.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/discovery.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/identity.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/mod.rs` +- 修改：`crates/persisting-pchronicle/src/store/catalog/source.rs` +- 修改：`crates/persisting-pchronicle/tests/atif_lance_corpus.rs` +- 修改：`crates/persisting-pchronicle/tests/s3_storage.rs` +- 修改：`crates/persisting-gateway/tests/markdown_trajectory.rs` + +**公开模块：** `model`、`document`、`storage`、`query`。解析器、wire DTO、Arrow codec、provider、manifest、lock 与 projection helper 改为 private 或 `pub(crate)`。 + +- [ ] 增加 compile-oriented API tests，只从四个公开模块导入批准的类型和函数；删除测试对旧根级/深层路径的使用。 +- [ ] 将字符串转换 API 限制到 AgenticMD/ATIF/OpenAI/ACTF；`CanonicalEvent` 与 Storyline Lance 通过类型化存储/document source，不再产生“不支持字符串 wire”的枚举分支。 +- [ ] 全量迁移 CLI、Gateway 和 pChronicle tests/benches；删除 `ChronicleFormat` 定义及 re-export，不添加 deprecated alias。 +- [ ] 运行： + +```bash +rg -n "ChronicleFormat|Agenticmd(Document|Block|Header)|pub mod (formats|convert|store|operations)" crates/persisting-pchronicle crates/persisting-pchronicle-cli crates/persisting-gateway +cargo check -p persisting-pchronicle-cli --all-targets --locked +cargo check -p persisting-gateway --all-targets --locked +``` + + `rg` 只允许命中迁移说明文字，不得命中代码标识符。 +- [ ] 提交：`refactor: narrow pchronicle public facade`。 + +### 任务 14：文档、严格 lint 与最终验收 + +**文件：** +- 修改：`crates/persisting-pchronicle/README.md` +- 修改：`crates/persisting-pchronicle/src/lib.rs` +- 修改：与新 public API 直接相关的 rustdoc +- 修改：`docs/superpowers/specs/2026-08-17-pchronicle-document-source-convergence-design.md`（仅状态和最终接口有名称差异时） + +- [ ] 更新 README/rustdoc：画清 Canonical Event → Storyline 单向投影、六种磁盘格式、无损边界和 provider 能力矩阵；不提 canonical v2 为现行设计。 +- [ ] 运行 `cargo fmt --all -- --check` 与 `git diff --check`。 +- [ ] 运行最终验收： + +```bash +cargo test -p persisting-pchronicle --no-default-features --locked +cargo test -p persisting-pchronicle --features lance-store --locked +cargo test -p persisting-pchronicle-cli --locked +cargo test -p persisting-gateway --locked +cargo clippy -p persisting-pchronicle --all-targets --features lance-store --locked -- -D warnings +RUSTDOCFLAGS="-D warnings" cargo doc -p persisting-pchronicle --no-deps --locked +``` + +- [ ] 运行静态验收： + +```bash +rg -n "_pchronicle_|ChronicleFormat|AtifDataSource|Agenticmd(Document|Block|Header)|fn transcode" \ + crates/persisting-pchronicle crates/persisting-pchronicle-cli crates/persisting-gateway +rg -n "messages_value\(\)\.ok|response_value\(\)\.ok|observation.*\.ok|sync_all\(\).*let _" \ + crates/persisting-pchronicle/src +``` + + 预期：无代码命中；若 fixture/data 中合法出现字面值，逐条人工解释，不修改数据来迎合检查。 +- [ ] 确认 `git status --short` 仅包含本计划范围内变更与用户原有未跟踪文件。 +- [ ] 提交：`docs: document pchronicle document source architecture`。 + +## 完成标准 + +全部 14 个任务及其 RED/GREEN 证据完成；六种 `DocumentFormat` 可通过统一读取入口打开；ATIF、ACTF、OpenAI Msg 经 Storyline Lance 的 JSON 数据模型级往返无损；AgenticMD 公共语义只暴露 Storyline；Canonical Event 和 Storyline 的 DataFusion 优化与能力声明一致；旧 ATIF DataSource、重复轨迹 DTO、serde transcode、`ChronicleFormat` 与公共 AgenticMD AST 均已删除；最终验收命令全部通过。 From b46371185ebfd97186d09eed5ea61141cd43cd69 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:05:03 +0800 Subject: [PATCH 17/65] refactor: define pchronicle document formats --- crates/persisting-pchronicle/src/error.rs | 51 +++++++++++- crates/persisting-pchronicle/src/format.rs | 93 ++++++++++++++++++++++ crates/persisting-pchronicle/src/lib.rs | 2 +- 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/crates/persisting-pchronicle/src/error.rs b/crates/persisting-pchronicle/src/error.rs index 3e5c5259..8ec49bb4 100644 --- a/crates/persisting-pchronicle/src/error.rs +++ b/crates/persisting-pchronicle/src/error.rs @@ -1,5 +1,8 @@ //! Error types for pChronicle. +use std::path::PathBuf; + +use crate::format::DocumentFormat; use thiserror::Error; #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] @@ -66,6 +69,27 @@ pub type Result = std::result::Result; #[derive(Debug, Error)] pub enum Error { + #[error("invalid {format} document (path={path:?}, location={location:?}): {message}")] + InvalidDocument { + format: DocumentFormat, + path: Option, + location: Option, + message: String, + }, + + #[error("{format} document cannot represent {stories} storylines")] + UnsupportedCardinality { + format: DocumentFormat, + stories: usize, + }, + + #[error("{format} source budget exceeded (path={path:?}, budget={budget})")] + SourceBudgetExceeded { + format: DocumentFormat, + path: Option, + budget: String, + }, + #[error("invalid ATIF: {0}")] InvalidAtif(String), @@ -104,7 +128,10 @@ pub enum Error { impl Error { pub fn code(&self) -> ErrorCode { match self { - Self::InvalidAtif(_) + Self::InvalidDocument { .. } + | Self::UnsupportedCardinality { .. } + | Self::SourceBudgetExceeded { .. } + | Self::InvalidAtif(_) | Self::DuplicateSession(_) | Self::DuplicateStep { .. } | Self::DuplicateToolCall { .. } @@ -116,3 +143,25 @@ impl Error { } } } + +#[cfg(test)] +mod tests { + use super::Error; + use crate::format::DocumentFormat; + + #[test] + fn invalid_document_error_identifies_format_and_location() { + let error = Error::InvalidDocument { + format: DocumentFormat::OpenaiMsg, + path: Some("sessions.json".into()), + location: Some("record[2].messages".into()), + message: "expected an array".into(), + }; + + let rendered = error.to_string(); + assert!(rendered.contains("openai-msg")); + assert!(rendered.contains("sessions.json")); + assert!(rendered.contains("record[2].messages")); + assert_eq!(error.code(), super::ErrorCode::InvalidInput); + } +} diff --git a/crates/persisting-pchronicle/src/format.rs b/crates/persisting-pchronicle/src/format.rs index 21692270..99c7e6b0 100644 --- a/crates/persisting-pchronicle/src/format.rs +++ b/crates/persisting-pchronicle/src/format.rs @@ -10,6 +10,72 @@ use crate::{Error, Result}; use std::fmt; use std::str::FromStr; +/// On-disk document formats understood by pChronicle. +/// +/// This enum describes physical representations. It does not imply that all +/// formats support the same read or write operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DocumentFormat { + /// Canonical append-only event facts stored in Lance. + CanonicalEvent, + /// Storyline runs, steps, tool calls, and objects stored in Lance. + Storyline, + /// Human-readable Storyline Markdown. + AgenticMd, + /// ATIF JSON, JSONL, or NDJSON. + Atif, + /// OpenAI message corpus JSON. + OpenaiMsg, + /// ACTF JSON. + Actf, +} + +impl DocumentFormat { + pub const ALL: &[Self] = &[ + Self::CanonicalEvent, + Self::Storyline, + Self::AgenticMd, + Self::Atif, + Self::OpenaiMsg, + Self::Actf, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::CanonicalEvent => "canonical-event", + Self::Storyline => "storyline", + Self::AgenticMd => "agenticmd", + Self::Atif => "atif", + Self::OpenaiMsg => "openai-msg", + Self::Actf => "actf", + } + } +} + +impl fmt::Display for DocumentFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for DocumentFormat { + type Err = Error; + + fn from_str(input: &str) -> Result { + match input.trim().to_ascii_lowercase().as_str() { + "canonical-event" => Ok(Self::CanonicalEvent), + "storyline" => Ok(Self::Storyline), + "agenticmd" => Ok(Self::AgenticMd), + "atif" => Ok(Self::Atif), + "openai-msg" => Ok(Self::OpenaiMsg), + "actf" => Ok(Self::Actf), + other => Err(Error::Other(format!( + "unknown document format '{other}'; expected canonical-event|storyline|agenticmd|atif|openai-msg|actf" + ))), + } + } +} + /// First-class trajectory storage formats. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ChronicleFormat { @@ -103,3 +169,30 @@ impl FromStr for ChronicleFormat { } } } + +#[cfg(test)] +mod tests { + use super::DocumentFormat; + use std::str::FromStr; + + #[test] + fn document_format_names_are_canonical_only() { + let cases = [ + ("canonical-event", DocumentFormat::CanonicalEvent), + ("storyline", DocumentFormat::Storyline), + ("agenticmd", DocumentFormat::AgenticMd), + ("atif", DocumentFormat::Atif), + ("openai-msg", DocumentFormat::OpenaiMsg), + ("actf", DocumentFormat::Actf), + ]; + + for (name, expected) in cases { + assert_eq!(DocumentFormat::from_str(name).unwrap(), expected); + assert_eq!(expected.to_string(), name); + } + + for alias in ["events", "lance", "md", "openai_msg", "session_steps"] { + assert!(DocumentFormat::from_str(alias).is_err(), "accepted {alias}"); + } + } +} diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index aa813b02..4a8d54a5 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -67,7 +67,7 @@ pub use discovery::{ drop_lifecycle_run_partitions, expand_story_locations, expand_story_locations_blocking, }; pub use error::{classify_error, Error, ErrorCode, Result}; -pub use format::ChronicleFormat; +pub use format::{ChronicleFormat, DocumentFormat}; pub use formats::{ agenticmd_body_byte_offset, append_subagent_refs_footer, block_speaker, detect_format, encode_agenticmd_block, encode_agenticmd_document, encode_agenticmd_preamble, From 40a03f982104c038e98d60f5631b326712bd3c1a Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:09:19 +0800 Subject: [PATCH 18/65] fix: reject malformed pchronicle documents --- .../src/agenticmd/codec.rs | 30 +++++- .../persisting-pchronicle/src/convert/atif.rs | 47 +++++++++- .../src/convert/openai_msg.rs | 91 ++++++++++++++++++- .../src/store/attempt_registry.rs | 4 +- .../src/store/events/manifest.rs | 4 +- .../src/store/run_control.rs | 4 +- .../src/store/storyline/mod.rs | 7 +- 7 files changed, 166 insertions(+), 21 deletions(-) diff --git a/crates/persisting-pchronicle/src/agenticmd/codec.rs b/crates/persisting-pchronicle/src/agenticmd/codec.rs index 1c1fda27..c9e2cfc3 100644 --- a/crates/persisting-pchronicle/src/agenticmd/codec.rs +++ b/crates/persisting-pchronicle/src/agenticmd/codec.rs @@ -20,7 +20,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{Error, Result}; +use crate::{DocumentFormat, Error, Result}; pub const AGENTICMD_FORMAT_NAME: &str = "agenticmd"; pub const AGENTICMD_FRONTMATTER_FORMAT: &str = "persisting"; @@ -229,11 +229,37 @@ fn split_frontmatter_with_offset(input: &str) -> Result<(BTreeMap let yaml = &rest[..end]; let after = &rest[end + "\n---".len()..]; let body = after.strip_prefix('\n').unwrap_or(after); - let map = serde_yaml::from_str::>(yaml).unwrap_or_default(); + let map = serde_yaml::from_str::>(yaml).map_err(|error| { + Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + path: None, + location: Some("frontmatter".into()), + message: error.to_string(), + } + })?; let body_offset = input.len() - body.len(); Ok((map, body, body_offset)) } +#[cfg(test)] +mod strict_frontmatter_tests { + use super::parse_agenticmd_document; + use crate::{DocumentFormat, Error}; + + #[test] + fn malformed_agenticmd_yaml_is_not_silently_replaced() { + let error = parse_agenticmd_document("---\nformat: [\n---\n\n").unwrap_err(); + assert!(matches!( + error, + Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + location: Some(ref location), + .. + } if location == "frontmatter" + )); + } +} + fn parse_blocks_with_spans(input: &str, base_offset: usize) -> Result> { if input.trim().is_empty() { return Ok(Vec::new()); diff --git a/crates/persisting-pchronicle/src/convert/atif.rs b/crates/persisting-pchronicle/src/convert/atif.rs index c52c5ef3..c388c429 100644 --- a/crates/persisting-pchronicle/src/convert/atif.rs +++ b/crates/persisting-pchronicle/src/convert/atif.rs @@ -4,7 +4,7 @@ use crate::atif::{AtifAgent, AtifObservation, AtifStep, AtifToolCall, AtifTrajec use crate::formats::storyline::{ StorylineAgent, StorylineDocument, StorylineToolCall, StorylineTurn, }; -use crate::Result; +use crate::{DocumentFormat, Error, Result}; const ATIF_TOOL_CALL_PROVENANCE_KEY: &str = "_pchronicle_atif_tool_call"; @@ -125,11 +125,21 @@ pub fn atif_to_storyline(traj: &AtifTrajectory) -> Result { pub fn storyline_to_atif(story: &StorylineDocument) -> Result { story.validate()?; let mut steps = Vec::new(); - for turn in &story.turns { + for (step_index, turn) in story.turns.iter().enumerate() { let observation = turn .observation .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()); + .map(|value| { + serde_json::from_value::(value.clone()).map_err(|error| { + Error::InvalidDocument { + format: DocumentFormat::Atif, + path: None, + location: Some(format!("step[{step_index}].observation")), + message: error.to_string(), + } + }) + }) + .transpose()?; let tool_calls = turn.tool_calls.as_ref().map(|calls| { calls @@ -232,3 +242,34 @@ pub fn storyline_to_atif(story: &StorylineDocument) -> Result { subagent_trajectories: None, }) } + +#[cfg(test)] +mod tests { + use super::{atif_to_storyline, storyline_to_atif}; + use crate::{AtifTrajectory, DocumentFormat, Error}; + + #[test] + fn malformed_atif_observation_is_not_silently_dropped() { + let trajectory = AtifTrajectory::from_json_str( + r#"{ + "schema_version":"ATIF-v1.7", + "session_id":"session-1", + "agent":{"name":"agent-1","version":"1"}, + "steps":[{"step_id":1,"source":"agent","message":"done"}] + }"#, + ) + .unwrap(); + let mut story = atif_to_storyline(&trajectory).unwrap(); + story.turns[0].observation = Some(serde_json::json!({"results":"not-an-array"})); + + let error = storyline_to_atif(&story).unwrap_err(); + assert!(matches!( + error, + Error::InvalidDocument { + format: DocumentFormat::Atif, + location: Some(ref location), + .. + } if location == "step[0].observation" + )); + } +} diff --git a/crates/persisting-pchronicle/src/convert/openai_msg.rs b/crates/persisting-pchronicle/src/convert/openai_msg.rs index 4017cdc4..1df485eb 100644 --- a/crates/persisting-pchronicle/src/convert/openai_msg.rs +++ b/crates/persisting-pchronicle/src/convert/openai_msg.rs @@ -3,15 +3,30 @@ use crate::convert::message_text; use crate::formats::openai_msg::{OpenaiMsgDocument, OpenaiMsgStep}; use crate::formats::storyline::{StorylineAgent, StorylineDocument, StorylineTurn}; -use crate::Result; +use crate::{DocumentFormat, Error, Result}; pub fn openai_msg_to_storyline(doc: &OpenaiMsgDocument) -> Result { let mut turns = Vec::new(); let mut next_id = 1i64; - for step in &doc.session_steps { - let messages = step.messages_value().ok(); - let response = step.response_value().ok().flatten(); + for (record_index, step) in doc.session_steps.iter().enumerate() { + let messages = Some( + step.messages_value() + .map_err(|error| Error::InvalidDocument { + format: DocumentFormat::OpenaiMsg, + path: None, + location: Some(format!("record[{record_index}].messages")), + message: error.to_string(), + })?, + ); + let response = step + .response_value() + .map_err(|error| Error::InvalidDocument { + format: DocumentFormat::OpenaiMsg, + path: None, + location: Some(format!("record[{record_index}].response")), + message: error.to_string(), + })?; let ts = Some(step.created_at.clone()).filter(|s| !s.is_empty()); if let Some(user_msg) = last_user_content(messages.as_ref()) { @@ -229,3 +244,71 @@ fn last_user_content(messages: Option<&serde_json::Value>) -> Option OpenaiMsgStep { + OpenaiMsgStep { + id: "record-7".into(), + session_id: "session-1".into(), + step_id: 7, + job_id: String::new(), + agent_id: "agent-1".into(), + group_id: String::new(), + env_name: String::new(), + llm_model: String::new(), + step_reward: 0.0, + reward: 0.0, + is_terminal: false, + is_truncated: false, + is_session_completed: false, + is_trainable: true, + created_at: String::new(), + messages: None, + response: None, + messages_json: Some("{".into()), + response_json: None, + env_state_json: None, + extensions_json: None, + capture_json: None, + run_bucket: String::new(), + call_id: String::new(), + source_export_id: None, + } + } + + #[test] + fn malformed_openai_messages_are_not_silently_dropped() { + let document = OpenaiMsgDocument::new("session-1", vec![step()]); + let error = openai_msg_to_storyline(&document).unwrap_err(); + assert!(matches!( + error, + Error::InvalidDocument { + format: DocumentFormat::OpenaiMsg, + location: Some(ref location), + .. + } if location == "record[0].messages" + )); + } + + #[test] + fn malformed_openai_response_is_not_silently_dropped() { + let mut record = step(); + record.messages_json = Some("[]".into()); + record.response_json = Some("{".into()); + let document = OpenaiMsgDocument::new("session-1", vec![record]); + let error = openai_msg_to_storyline(&document).unwrap_err(); + assert!(matches!( + error, + Error::InvalidDocument { + format: DocumentFormat::OpenaiMsg, + location: Some(ref location), + .. + } if location == "record[0].response" + )); + } +} diff --git a/crates/persisting-pchronicle/src/store/attempt_registry.rs b/crates/persisting-pchronicle/src/store/attempt_registry.rs index 6baada36..62af6a4e 100644 --- a/crates/persisting-pchronicle/src/store/attempt_registry.rs +++ b/crates/persisting-pchronicle/src/store/attempt_registry.rs @@ -342,9 +342,7 @@ fn write_local_record(path: &Path, record: &AttemptRecord) -> anyhow::Result<()> file.write_all(&serde_json::to_vec_pretty(record)?)?; file.sync_all()?; std::fs::rename(&temporary, path)?; - if let Ok(directory) = File::open(parent) { - let _ = directory.sync_all(); - } + File::open(parent)?.sync_all()?; Ok(()) } diff --git a/crates/persisting-pchronicle/src/store/events/manifest.rs b/crates/persisting-pchronicle/src/store/events/manifest.rs index a8a8a5c6..9346e171 100644 --- a/crates/persisting-pchronicle/src/store/events/manifest.rs +++ b/crates/persisting-pchronicle/src/store/events/manifest.rs @@ -524,9 +524,7 @@ fn write_local_manifest(path: &Path, manifest: &EventManifest) -> Result<()> { file.write_all(&serde_json::to_vec_pretty(manifest)?)?; file.sync_all()?; std::fs::rename(&temporary, path)?; - if let Ok(directory) = File::open(parent) { - let _ = directory.sync_all(); - } + File::open(parent)?.sync_all()?; Ok(()) } diff --git a/crates/persisting-pchronicle/src/store/run_control.rs b/crates/persisting-pchronicle/src/store/run_control.rs index ded02454..ad2fdc78 100644 --- a/crates/persisting-pchronicle/src/store/run_control.rs +++ b/crates/persisting-pchronicle/src/store/run_control.rs @@ -506,9 +506,7 @@ fn write_local_record(path: &Path, record: &RunControlRecord) -> anyhow::Result< file.write_all(&serde_json::to_vec_pretty(record)?)?; file.sync_all()?; std::fs::rename(&temporary, path)?; - if let Ok(directory) = File::open(parent) { - let _ = directory.sync_all(); - } + File::open(parent)?.sync_all()?; Ok(()) } diff --git a/crates/persisting-pchronicle/src/store/storyline/mod.rs b/crates/persisting-pchronicle/src/store/storyline/mod.rs index f0d0836e..1446cb93 100644 --- a/crates/persisting-pchronicle/src/store/storyline/mod.rs +++ b/crates/persisting-pchronicle/src/store/storyline/mod.rs @@ -1018,9 +1018,10 @@ async fn write_local_current(path: PathBuf, contents: Vec) -> Result<()> { .with_context(|| format!("sync Storyline CURRENT temp {}", temporary.display()))?; std::fs::rename(&temporary, &path) .with_context(|| format!("publish Storyline CURRENT {}", path.display()))?; - if let Ok(directory) = File::open(parent) { - let _ = directory.sync_all(); - } + File::open(parent) + .with_context(|| format!("open Storyline root {} for sync", parent.display()))? + .sync_all() + .with_context(|| format!("sync Storyline root {}", parent.display()))?; Ok(()) }) .await From 363c466d9c95e5104dcaf69c64dfd6d9fb6d06a8 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:18:02 +0800 Subject: [PATCH 19/65] feat: align storyline with atif presence semantics --- .../src/agenticmd/convert.rs | 2 + crates/persisting-pchronicle/src/atif.rs | 8 +- .../persisting-pchronicle/src/convert/actf.rs | 3 + .../persisting-pchronicle/src/convert/atif.rs | 87 ++++++++++++------- .../src/convert/events.rs | 2 + .../src/convert/openai_msg.rs | 2 + .../persisting-pchronicle/src/formats/mod.rs | 4 +- .../src/formats/openai_corpus.rs | 4 + .../src/formats/storyline.rs | 80 +++++++++++++++-- crates/persisting-pchronicle/src/lib.rs | 14 +-- .../src/store/catalog/tests.rs | 2 + .../src/store/storyline/model.rs | 22 ++++- .../src/store/storyline/rows.rs | 35 +++++++- .../src/store/storyline/tests.rs | 3 + crates/persisting-pchronicle/src/tests.rs | 10 ++- 15 files changed, 221 insertions(+), 57 deletions(-) diff --git a/crates/persisting-pchronicle/src/agenticmd/convert.rs b/crates/persisting-pchronicle/src/agenticmd/convert.rs index 7a5fdd83..04f3554e 100644 --- a/crates/persisting-pchronicle/src/agenticmd/convert.rs +++ b/crates/persisting-pchronicle/src/agenticmd/convert.rs @@ -84,7 +84,9 @@ pub fn agenticmd_to_storyline(doc: &AgenticmdDocument) -> Result, + /// Inline result. ATIF distinguishes an omitted result from explicit null. + #[serde(default, skip_serializing_if = "FieldPresence::is_missing")] + pub result: FieldPresence, #[serde(default, skip_serializing_if = "Option::is_none")] pub extra: Option, } diff --git a/crates/persisting-pchronicle/src/convert/actf.rs b/crates/persisting-pchronicle/src/convert/actf.rs index 97be8601..f30a25dd 100644 --- a/crates/persisting-pchronicle/src/convert/actf.rs +++ b/crates/persisting-pchronicle/src/convert/actf.rs @@ -163,6 +163,7 @@ fn attempt_to_storyline( tool_call_id: call.id.clone(), function_name: actf_tool_name(call), arguments: actf_tool_arguments(call), + result: Default::default(), duration_ms: if step.tools.len() == 1 { step.metric.env_action_ms.as_f64().map(|value| value as i64) } else { @@ -219,7 +220,9 @@ fn attempt_to_storyline( document.task_id.clone() }; Ok(StorylineDocument { + schema_version: None, run_id: Some(document.task_id.clone()), + attempt_id: None, session_id, agent: StorylineAgent { id: "actf-agent".into(), diff --git a/crates/persisting-pchronicle/src/convert/atif.rs b/crates/persisting-pchronicle/src/convert/atif.rs index c388c429..6c2b5d6e 100644 --- a/crates/persisting-pchronicle/src/convert/atif.rs +++ b/crates/persisting-pchronicle/src/convert/atif.rs @@ -6,8 +6,6 @@ use crate::formats::storyline::{ }; use crate::{DocumentFormat, Error, Result}; -const ATIF_TOOL_CALL_PROVENANCE_KEY: &str = "_pchronicle_atif_tool_call"; - fn timing_from_metrics(metrics: &Option) -> (Option, Option) { let Some(m) = metrics else { return (None, None); @@ -46,19 +44,13 @@ pub fn atif_to_storyline(traj: &AtifTrajectory) -> Result { .as_ref() .and_then(|x| x.get("duration_ms")) .and_then(|v| v.as_i64()); - let extra = Some(serde_json::json!({ - ATIF_TOOL_CALL_PROVENANCE_KEY: { - "result_present": c.result.is_some(), - "result": c.result, - "extra": c.extra, - } - })); StorylineToolCall { tool_call_id: c.tool_call_id.clone(), function_name: c.function_name.clone(), arguments: c.arguments.clone(), + result: c.result.clone(), duration_ms, - extra, + extra: c.extra.clone(), } }) .collect::>() @@ -98,7 +90,9 @@ pub fn atif_to_storyline(traj: &AtifTrajectory) -> Result { } Ok(StorylineDocument { + schema_version: Some(traj.schema_version.clone()), run_id: traj.trajectory_id.clone(), + attempt_id: None, session_id, agent: StorylineAgent { id: traj.agent.name.clone(), @@ -145,25 +139,7 @@ pub fn storyline_to_atif(story: &StorylineDocument) -> Result { calls .iter() .map(|c| { - let provenance = c - .extra - .as_ref() - .and_then(|extra| extra.get(ATIF_TOOL_CALL_PROVENANCE_KEY)); - let result = provenance - .filter(|value| { - value.get("result_present").and_then(|v| v.as_bool()) == Some(true) - }) - .and_then(|value| value.get("result")) - .cloned(); - let mut extra = if let Some(provenance) = provenance { - provenance - .get("extra") - .filter(|value| !value.is_null()) - .cloned() - } else { - c.extra.clone() - } - .unwrap_or(serde_json::json!({})); + let mut extra = c.extra.clone().unwrap_or(serde_json::json!({})); if let Some(ms) = c.duration_ms { if let Some(obj) = extra.as_object_mut() { obj.insert("duration_ms".into(), serde_json::json!(ms)); @@ -178,7 +154,7 @@ pub fn storyline_to_atif(story: &StorylineDocument) -> Result { tool_call_id: c.tool_call_id.clone(), function_name: c.function_name.clone(), arguments: c.arguments.clone(), - result, + result: c.result.clone(), extra, } }) @@ -220,7 +196,10 @@ pub fn storyline_to_atif(story: &StorylineDocument) -> Result { } Ok(AtifTrajectory { - schema_version: "ATIF-v1.7".into(), + schema_version: story + .schema_version + .clone() + .unwrap_or_else(|| "ATIF-v1.7".into()), session_id: Some(story.session_id.clone()), trajectory_id: story.run_id.clone(), agent: AtifAgent { @@ -246,7 +225,7 @@ pub fn storyline_to_atif(story: &StorylineDocument) -> Result { #[cfg(test)] mod tests { use super::{atif_to_storyline, storyline_to_atif}; - use crate::{AtifTrajectory, DocumentFormat, Error}; + use crate::{AtifTrajectory, DocumentFormat, Error, FieldPresence}; #[test] fn malformed_atif_observation_is_not_silently_dropped() { @@ -272,4 +251,48 @@ mod tests { } if location == "step[0].observation" )); } + + #[test] + fn atif_tool_result_presence_round_trips_without_provenance() { + let trajectory = AtifTrajectory::from_json_str( + r#"{ + "schema_version":"ATIF-v1.7", + "session_id":"session-1", + "agent":{"name":"agent-1","version":"1"}, + "steps":[{ + "step_id":1, + "source":"agent", + "message":"done", + "tool_calls":[ + {"tool_call_id":"missing","function_name":"a","arguments":{}}, + {"tool_call_id":"null","function_name":"b","arguments":{},"result":null}, + {"tool_call_id":"value","function_name":"c","arguments":{},"result":{"ok":true}} + ] + }] + }"#, + ) + .unwrap(); + + let story = atif_to_storyline(&trajectory).unwrap(); + assert_eq!(story.schema_version.as_deref(), Some("ATIF-v1.7")); + let calls = story.turns[0].tool_calls.as_ref().unwrap(); + assert_eq!(calls[0].result, FieldPresence::Missing); + assert_eq!(calls[1].result, FieldPresence::Null); + assert_eq!( + calls[2].result, + FieldPresence::Value(serde_json::json!({"ok": true})) + ); + assert!(calls.iter().all(|call| { + !call + .extra + .as_ref() + .is_some_and(|extra| extra.to_string().contains("_pchronicle_")) + })); + + let encoded = serde_json::to_value(storyline_to_atif(&story).unwrap()).unwrap(); + let calls = encoded["steps"][0]["tool_calls"].as_array().unwrap(); + assert!(calls[0].get("result").is_none()); + assert_eq!(calls[1]["result"], serde_json::Value::Null); + assert_eq!(calls[2]["result"], serde_json::json!({"ok": true})); + } } diff --git a/crates/persisting-pchronicle/src/convert/events.rs b/crates/persisting-pchronicle/src/convert/events.rs index 1e490155..fe92f766 100644 --- a/crates/persisting-pchronicle/src/convert/events.rs +++ b/crates/persisting-pchronicle/src/convert/events.rs @@ -290,7 +290,9 @@ fn events_to_storyline_unchecked(events: &[EventRecord]) -> Result Result) -> Option> { tool_call_id, function_name, arguments, + result: Default::default(), duration_ms: None, extra: Some(Value::Object(call.clone())), }) @@ -694,6 +697,7 @@ fn parse_embedded_tool_call( tool_call_id: format!("embedded-{step_id}-{name}"), function_name: name.to_string(), arguments: Value::Object(arguments), + result: Default::default(), duration_ms: None, extra: Some(json!({"encoding":"embedded_text"})), }]) diff --git a/crates/persisting-pchronicle/src/formats/storyline.rs b/crates/persisting-pchronicle/src/formats/storyline.rs index c7664299..3399c2a0 100644 --- a/crates/persisting-pchronicle/src/formats/storyline.rs +++ b/crates/persisting-pchronicle/src/formats/storyline.rs @@ -9,10 +9,54 @@ use serde_json::Value; use crate::{Error, Result}; +/// Presence semantics for interchange fields where missing and explicit null +/// carry different meanings. +#[derive(Debug, Clone, Default, PartialEq)] +pub enum FieldPresence { + #[default] + Missing, + Null, + Value(T), +} + +impl FieldPresence { + pub fn is_missing(&self) -> bool { + matches!(self, Self::Missing) + } +} + +impl Serialize for FieldPresence { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::Missing | Self::Null => serializer.serialize_none(), + Self::Value(value) => value.serialize(serializer), + } + } +} + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for FieldPresence { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + Ok(match Option::::deserialize(deserializer)? { + Some(value) => Self::Value(value), + None => Self::Null, + }) + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct StorylineDocument { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema_version: Option, #[serde(rename = "run", default, skip_serializing_if = "Option::is_none")] pub run_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attempt_id: Option, /// Session id (≈ ATIF / Capture `session_id`). Wire key: `session`. #[serde(rename = "session")] pub session_id: String, @@ -139,6 +183,8 @@ pub struct StorylineToolCall { pub function_name: String, #[serde(rename = "args")] pub arguments: Value, + #[serde(default, skip_serializing_if = "FieldPresence::is_missing")] + pub result: FieldPresence, /// Tool execution wall time in milliseconds. #[serde(default, skip_serializing_if = "Option::is_none")] pub duration_ms: Option, @@ -150,7 +196,9 @@ impl StorylineDocument { pub fn new(session_id: impl Into, agent_id: impl Into) -> Self { let agent_id = agent_id.into(); Self { + schema_version: None, run_id: None, + attempt_id: None, session_id: session_id.into(), agent: StorylineAgent { id: agent_id.clone(), @@ -219,10 +267,32 @@ mod tests { } #[test] - fn storyline_wire_has_no_schema_marker() { - let document = StorylineDocument::new("session-1", "agent-1"); - let value = serde_json::to_value(document).unwrap(); - assert!(value.get("spec").is_none()); - assert!(value.get("schema_version").is_none()); + fn tool_result_presence_distinguishes_missing_null_and_value() { + let base = serde_json::json!({"tcid":"call-1","fn":"lookup","args":{}}); + + let missing: StorylineToolCall = serde_json::from_value(base.clone()).unwrap(); + assert_eq!(missing.result, FieldPresence::Missing); + assert!(serde_json::to_value(missing) + .unwrap() + .get("result") + .is_none()); + + let mut null = base.clone(); + null["result"] = Value::Null; + let null: StorylineToolCall = serde_json::from_value(null).unwrap(); + assert_eq!(null.result, FieldPresence::Null); + assert_eq!(serde_json::to_value(null).unwrap()["result"], Value::Null); + + let mut value = base; + value["result"] = serde_json::json!({"answer": 42}); + let value: StorylineToolCall = serde_json::from_value(value).unwrap(); + assert_eq!( + value.result, + FieldPresence::Value(serde_json::json!({"answer": 42})) + ); + assert_eq!( + serde_json::to_value(value).unwrap()["result"], + serde_json::json!({"answer": 42}) + ); } } diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index 4a8d54a5..32728cd7 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -77,13 +77,13 @@ pub use formats::{ strip_subagent_footer_from_body, validate_agenticmd_block, validate_speaker, validate_type_name, AgenticmdBlock, AgenticmdBlockSpan, AgenticmdClientMeta, AgenticmdDocument, AgenticmdHeader, AgenticmdSessionFrontmatter, ChronicleEventRecordExt, EventIdentity, - EventRecord, EventsDocument, LlmCandidate, LlmContentPart, LlmExtensions, LlmGenerationParams, - LlmImageSource, LlmMessage, LlmProtocol, LlmRequest, LlmRequestEventPayload, LlmResponse, - LlmResponseEventPayload, LlmResponseFormat, LlmRole, LlmStreamEvent, LlmToolChoice, - LlmToolChoiceMode, LlmToolDefinition, LlmUsage, OpenaiMsgCorpusReader, OpenaiMsgDocument, - OpenaiMsgStep, RecoveredOpenaiMsgFile, StoryLink, StorylineAgent, StorylineDocument, - StorylineToolCall, StorylineTurn, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FORMAT_NAME, - AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, + EventRecord, EventsDocument, FieldPresence, LlmCandidate, LlmContentPart, LlmExtensions, + LlmGenerationParams, LlmImageSource, LlmMessage, LlmProtocol, LlmRequest, + LlmRequestEventPayload, LlmResponse, LlmResponseEventPayload, LlmResponseFormat, LlmRole, + LlmStreamEvent, LlmToolChoice, LlmToolChoiceMode, LlmToolDefinition, LlmUsage, + OpenaiMsgCorpusReader, OpenaiMsgDocument, OpenaiMsgStep, RecoveredOpenaiMsgFile, StoryLink, + StorylineAgent, StorylineDocument, StorylineToolCall, StorylineTurn, AGENTICMD_BLOCK_LAYOUT, + AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, }; pub use formats::{ is_lossless_openai_storyline, parse_openai_msg_corpus_value, recover_openai_msg_files, diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index 4545b8db..abe48e8d 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -18,7 +18,9 @@ fn write_openai_source(path: &Path, event_id: &str) -> Result<()> { fn storyline(session_id: &str, run_id: &str) -> StorylineDocument { StorylineDocument { + schema_version: None, run_id: Some(run_id.into()), + attempt_id: None, session_id: session_id.into(), agent: StorylineAgent { id: "agent".into(), diff --git a/crates/persisting-pchronicle/src/store/storyline/model.rs b/crates/persisting-pchronicle/src/store/storyline/model.rs index 1350cfaa..bd05dd12 100644 --- a/crates/persisting-pchronicle/src/store/storyline/model.rs +++ b/crates/persisting-pchronicle/src/store/storyline/model.rs @@ -9,7 +9,9 @@ use std::collections::{BTreeMap, HashSet}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{Error, Result, StoryLink, StorylineDocument, StorylineToolCall, StorylineTurn}; +use crate::{ + Error, FieldPresence, Result, StoryLink, StorylineDocument, StorylineToolCall, StorylineTurn, +}; pub const STORY_RUNS_TABLE: &str = "runs"; pub const STORY_STEPS_TABLE: &str = "steps"; @@ -17,10 +19,12 @@ pub const STORY_TOOL_CALLS_TABLE: &str = "tool_calls"; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct StoryRunRow { + pub schema_version: Option, pub run_id: String, /// Whether `run_id` was present in the source document. When false, /// `run_id` contains the effective value (`session_id`) used for joins. pub run_id_explicit: bool, + pub attempt_id: Option, pub session_id: String, pub agent_id: String, pub agent_name: Option, @@ -70,6 +74,7 @@ pub struct StoryToolCallRow { pub tool_call_id: String, pub function_name: String, pub arguments: Value, + pub result: FieldPresence, pub results: Vec, pub duration_ms: Option, pub extra: Option, @@ -108,8 +113,10 @@ pub fn split_storyline(story: &StorylineDocument) -> Result { .clone() .unwrap_or_else(|| story.session_id.clone()); let run = StoryRunRow { + schema_version: story.schema_version.clone(), run_id: run_id.clone(), run_id_explicit: story.run_id.is_some(), + attempt_id: story.attempt_id.clone(), session_id: story.session_id.clone(), agent_id: story.agent.id.clone(), agent_name: story.agent.name.clone(), @@ -174,6 +181,7 @@ pub fn split_storyline(story: &StorylineDocument) -> Result { tool_call_id: call.tool_call_id.clone(), function_name: call.function_name.clone(), arguments: call.arguments.clone(), + result: call.result.clone(), results: Vec::new(), duration_ms: call.duration_ms, extra: call.extra.clone(), @@ -278,6 +286,7 @@ pub fn reconstruct_storyline(tables: StorylineTables) -> Result Result Field { Field::new(name, data_type, nullable) @@ -18,8 +19,10 @@ fn field(name: &str, data_type: DataType, nullable: bool) -> Field { pub fn story_runs_arrow_schema() -> Arc { Arc::new(ArrowSchema::new(vec![ + field("schema_version", DataType::Utf8, true), field("run_id", DataType::Utf8, false), field("run_id_explicit", DataType::Boolean, false), + field("attempt_id", DataType::Utf8, true), field("session_id", DataType::Utf8, false), field("agent_id", DataType::Utf8, false), field("agent_name", DataType::Utf8, true), @@ -72,6 +75,8 @@ pub fn story_tool_calls_arrow_schema() -> Arc { field("tool_call_id", DataType::Utf8, false), field("function_name", DataType::Utf8, false), field("arguments_json", DataType::Utf8, false), + field("result_present", DataType::Boolean, false), + field("result_json", DataType::Utf8, true), field("results_json", DataType::Utf8, false), field("duration_ms", DataType::Int64, true), field("extra_json", DataType::Utf8, true), @@ -124,10 +129,12 @@ pub fn story_runs_to_batch(rows: &[StoryRunRow]) -> Result { RecordBatch::try_new( story_runs_arrow_schema(), vec![ + Arc::new(opt_utf8(rows.iter().map(|r| r.schema_version.as_deref()))), Arc::new(req_utf8(rows.iter().map(|r| r.run_id.as_str()))), Arc::new(BooleanArray::from( rows.iter().map(|r| r.run_id_explicit).collect::>(), )), + Arc::new(opt_utf8(rows.iter().map(|r| r.attempt_id.as_deref()))), Arc::new(req_utf8(rows.iter().map(|r| r.session_id.as_str()))), Arc::new(req_utf8(rows.iter().map(|r| r.agent_id.as_str()))), Arc::new(opt_utf8(rows.iter().map(|r| r.agent_name.as_deref()))), @@ -250,6 +257,19 @@ pub fn story_tool_calls_to_batch(rows: &[StoryToolCallRow]) -> Result>()?, )), + Arc::new(BooleanArray::from( + rows.iter() + .map(|r| !r.result.is_missing()) + .collect::>(), + )), + Arc::new(opt_utf8_owned( + rows.iter() + .map(|r| match &r.result { + FieldPresence::Value(value) => json(value).map(Some), + FieldPresence::Missing | FieldPresence::Null => Ok(None), + }) + .collect::>>()?, + )), Arc::new(req_utf8_owned( rows.iter() .map(|r| json(&r.results)) @@ -351,8 +371,10 @@ pub fn story_runs_from_batch(batch: &RecordBatch) -> Result> { (0..batch.num_rows()) .map(|row| { Ok(StoryRunRow { + schema_version: string_at(batch, "schema_version", row)?, run_id: required_string_at(batch, "run_id", row)?, run_id_explicit: required_bool_at(batch, "run_id_explicit", row)?, + attempt_id: string_at(batch, "attempt_id", row)?, session_id: required_string_at(batch, "session_id", row)?, agent_id: required_string_at(batch, "agent_id", row)?, agent_name: string_at(batch, "agent_name", row)?, @@ -419,6 +441,14 @@ pub fn story_tool_calls_from_batch(batch: &RecordBatch) -> Result FieldPresence::Value(parse_json(value, "result_json")?), + None => FieldPresence::Null, + } + } else { + FieldPresence::Missing + }, results: parse_json( required_string_at(batch, "results_json", row)?, "results_json", @@ -436,9 +466,9 @@ mod tests { #[test] fn empty_batches_keep_all_three_schemas() { - assert_eq!(story_runs_to_batch(&[]).unwrap().num_columns(), 15); + assert_eq!(story_runs_to_batch(&[]).unwrap().num_columns(), 17); assert_eq!(story_steps_to_batch(&[]).unwrap().num_columns(), 18); - assert_eq!(story_tool_calls_to_batch(&[]).unwrap().num_columns(), 10); + assert_eq!(story_tool_calls_to_batch(&[]).unwrap().num_columns(), 12); } #[test] @@ -512,6 +542,7 @@ mod tests { tool_call_id: "c".into(), function_name: "lookup".into(), arguments: serde_json::json!({"q": "x"}), + result: FieldPresence::Null, results: vec![serde_json::json!({"source_call_id": "c", "content": "y"})], duration_ms: Some(8), extra: None, diff --git a/crates/persisting-pchronicle/src/store/storyline/tests.rs b/crates/persisting-pchronicle/src/store/storyline/tests.rs index 2d89e2cf..197719e3 100644 --- a/crates/persisting-pchronicle/src/store/storyline/tests.rs +++ b/crates/persisting-pchronicle/src/store/storyline/tests.rs @@ -33,7 +33,9 @@ async fn put_remote_object(uri: &str, relative: &str, contents: &[u8]) { fn story(session_id: &str) -> StorylineDocument { StorylineDocument { + schema_version: None, run_id: Some("run-1".into()), + attempt_id: None, session_id: session_id.into(), agent: StorylineAgent { id: "agent-1".into(), @@ -80,6 +82,7 @@ fn story(session_id: &str) -> StorylineDocument { tool_call_id: "call-1".into(), function_name: "lookup".into(), arguments: serde_json::json!({"symbol": "ACME"}), + result: Default::default(), duration_ms: Some(12), extra: None, }]), diff --git a/crates/persisting-pchronicle/src/tests.rs b/crates/persisting-pchronicle/src/tests.rs index 24b770fb..4adf99b8 100644 --- a/crates/persisting-pchronicle/src/tests.rs +++ b/crates/persisting-pchronicle/src/tests.rs @@ -48,14 +48,14 @@ fn sample_traj() -> AtifTrajectory { tool_call_id: "call_price_1".into(), function_name: "financial_search".into(), arguments: json!({"ticker":"GOOGL","metric":"price"}), - result: Some(json!({"price": 185.35})), + result: crate::FieldPresence::Value(json!({"price": 185.35})), extra: Some(json!({"duration_ms": 42})), }, AtifToolCall { tool_call_id: "call_volume_2".into(), function_name: "financial_search".into(), arguments: json!({"ticker":"GOOGL","metric":"volume"}), - result: None, + result: crate::FieldPresence::Missing, extra: Some(json!({"duration_ms": 37})), }, ]), @@ -143,7 +143,7 @@ fn atif_storyline_hub_roundtrip() { assert_eq!(back.steps[1].tool_calls.as_ref().unwrap().len(), 2); assert_eq!( back.steps[1].tool_calls.as_ref().unwrap()[0].result, - Some(serde_json::json!({"price": 185.35})) + crate::FieldPresence::Value(serde_json::json!({"price": 185.35})) ); assert_eq!( back.steps[1] @@ -602,7 +602,7 @@ fn storyline_wire_uses_short_keys() { assert!(!out.contains(r#""agt""#)); assert!(!out.contains(r#""fm""#)); assert!(!out.contains(r#""kids""#)); - assert!(!out.contains(r#""schema_version""#)); + assert!(out.contains(r#""schema_version": "ATIF-v1.7""#)); assert!(!out.contains(r#""source""#)); assert!(!out.contains(r#""message""#)); } @@ -912,7 +912,9 @@ fn storyline_to_events_assigns_call_id_for_paired_turns() { use crate::formats::storyline::{StorylineAgent, StorylineDocument, StorylineTurn}; use serde_json::json; let story = StorylineDocument { + schema_version: None, run_id: None, + attempt_id: None, session_id: "s-pair".into(), agent: StorylineAgent { id: "a1".into(), From 09bf2d95f691059cd62ad81d9d30345269441180 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:25:15 +0800 Subject: [PATCH 20/65] fix: make actf residual roundtrips authoritative --- .../persisting-pchronicle/src/convert/actf.rs | 415 +++++++++++++----- 1 file changed, 295 insertions(+), 120 deletions(-) diff --git a/crates/persisting-pchronicle/src/convert/actf.rs b/crates/persisting-pchronicle/src/convert/actf.rs index f30a25dd..74adc0cc 100644 --- a/crates/persisting-pchronicle/src/convert/actf.rs +++ b/crates/persisting-pchronicle/src/convert/actf.rs @@ -5,16 +5,15 @@ use std::collections::BTreeMap; use serde_json::{json, Map, Value}; use crate::formats::actf::{ - ActfAssistantContent, ActfAttempt, ActfDocument, ActfMetric, ActfObservation, ActfStep, - ActfToolCall, ActfTrajectory, ACTF_SCHEMA_VERSION, + ActfAttempt, ActfDocument, ActfObservation, ActfStep, ActfToolCall, ActfTrajectory, + ACTF_SCHEMA_VERSION, }; use crate::formats::storyline::{ StorylineAgent, StorylineDocument, StorylineToolCall, StorylineTurn, }; use crate::{Error, Result}; -const ACTF_PROVENANCE_KEY: &str = "_pchronicle_actf"; -const ACTF_STEP_KEY: &str = "_pchronicle_actf_step"; +const ACTF_EXTENSION_KEY: &str = "persisting.dev/actf/v1"; pub fn actf_to_storyline(document: &ActfDocument) -> Result { let mut stories = actf_to_storylines(document)?; @@ -56,73 +55,84 @@ pub fn storylines_to_actf(stories: &[StorylineDocument]) -> Result "ACTF conversion requires at least one Storyline".into(), )); } - let provenance_count = stories + let residual_count = stories .iter() - .filter(|story| provenance(story).is_some()) + .filter(|story| residual(story).is_some()) .count(); - if provenance_count == 0 { + if residual_count == 0 { if stories.len() != 1 { return Err(Error::Other( - "synthesizing ACTF without provenance requires one Storyline".into(), + "synthesizing ACTF without residual metadata requires one Storyline".into(), )); } return synthesize_actf(&stories[0]); } - if provenance_count != stories.len() { + if residual_count != stories.len() { return Err(Error::Other( - "cannot mix ACTF-provenance and unrelated Storylines".into(), + "cannot mix ACTF residual and unrelated Storylines".into(), )); } - let first = provenance(&stories[0]) - .ok_or_else(|| Error::Other("ACTF provenance disappeared during conversion".into()))?; + let first = residual(&stories[0]) + .ok_or_else(|| Error::Other("ACTF residual disappeared during conversion".into()))?; let root_value = first .get("root") .and_then(Value::as_object) - .ok_or_else(|| Error::Other("ACTF provenance missing root metadata".into()))? + .ok_or_else(|| Error::Other("ACTF residual missing root metadata".into()))? .clone(); let mut attempts = Map::new(); for story in stories { story.validate()?; - let metadata = provenance(story) - .ok_or_else(|| Error::Other("ACTF provenance disappeared during conversion".into()))?; + let metadata = residual(story) + .ok_or_else(|| Error::Other("ACTF residual disappeared during conversion".into()))?; if metadata.get("root").and_then(Value::as_object) != Some(&root_value) { return Err(Error::Other( - "ACTF Storylines have conflicting root metadata".into(), + "ACTF Storylines have conflicting root residual".into(), )); } let attempt_id = metadata .get("attempt_id") .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or_else(|| Error::Other("ACTF provenance missing attempt_id".into()))?; + .ok_or_else(|| Error::Other("ACTF residual missing attempt_id".into()))?; let mut attempt = metadata .get("attempt") .and_then(Value::as_object) .cloned() - .ok_or_else(|| Error::Other("ACTF provenance missing attempt metadata".into()))?; + .ok_or_else(|| Error::Other("ACTF residual missing attempt metadata".into()))?; let mut trajectory = metadata .get("trajectory") .and_then(Value::as_object) .cloned() - .ok_or_else(|| Error::Other("ACTF provenance missing trajectory metadata".into()))?; + .ok_or_else(|| Error::Other("ACTF residual missing trajectory metadata".into()))?; let steps = story .turns .iter() - .map(|turn| { - turn.extra - .as_ref() - .and_then(|extra| extra.get(ACTF_STEP_KEY)) - .cloned() - .ok_or_else(|| { - Error::Other(format!( - "Storyline '{}' step {} lacks ACTF lossless step metadata", - story.session_id, turn.id - )) - }) - }) + .map(storyline_step_value) .collect::>>()?; trajectory.insert("steps".into(), Value::Array(steps)); + let metrics = story.final_metrics.as_ref().and_then(Value::as_object); + attempt.insert( + "correct".into(), + metrics + .and_then(|value| value.get("correct")) + .cloned() + .unwrap_or(Value::Bool(false)), + ); + attempt.insert( + "score".into(), + metrics + .and_then(|value| value.get("score")) + .cloned() + .unwrap_or(Value::Null), + ); + attempt.insert( + "status".into(), + metrics + .and_then(|value| value.get("status")) + .cloned() + .unwrap_or_else(|| Value::String("completed".into())), + ); attempt.insert("trajectory".into(), Value::Object(trajectory)); if attempts .insert(attempt_id.to_string(), Value::Object(attempt)) @@ -135,6 +145,24 @@ pub fn storylines_to_actf(stories: &[StorylineDocument]) -> Result } let mut root = root_value; + root.insert( + "task_id".into(), + Value::String( + stories[0] + .run_id + .clone() + .unwrap_or_else(|| stories[0].session_id.clone()), + ), + ); + root.insert( + "correct".into(), + stories[0] + .final_metrics + .as_ref() + .and_then(|value| value.get("task_correct")) + .cloned() + .unwrap_or(Value::Bool(false)), + ); root.insert("attempts".into(), Value::Object(attempts)); let document: ActfDocument = serde_json::from_value(Value::Object(root))?; document.validate()?; @@ -142,7 +170,7 @@ pub fn storylines_to_actf(stories: &[StorylineDocument]) -> Result } pub fn is_actf_storyline(story: &StorylineDocument) -> bool { - provenance(story).is_some() + residual(story).is_some() } fn attempt_to_storyline( @@ -152,27 +180,33 @@ fn attempt_to_storyline( root_metadata: &Value, multiple_attempts: bool, ) -> Result { - let attempt_metadata = attempt_metadata(attempt)?; - let trajectory_metadata = trajectory_metadata(&attempt.trajectory)?; + let attempt_metadata = attempt_residual(attempt)?; + let trajectory_metadata = trajectory_residual(&attempt.trajectory)?; let mut turns = Vec::with_capacity(attempt.trajectory.steps.len()); for step in &attempt.trajectory.steps { - let tool_calls = (!step.tools.is_empty()).then(|| { - step.tools - .iter() - .map(|call| StorylineToolCall { - tool_call_id: call.id.clone(), - function_name: actf_tool_name(call), - arguments: actf_tool_arguments(call), - result: Default::default(), - duration_ms: if step.tools.len() == 1 { - step.metric.env_action_ms.as_f64().map(|value| value as i64) - } else { - None - }, - extra: Some(json!({"actf_type": call.kind, "actf_extra": call.extra})), - }) - .collect::>() - }); + let tool_calls = (!step.tools.is_empty()) + .then(|| { + step.tools + .iter() + .map(|call| { + Ok(StorylineToolCall { + tool_call_id: call.id.clone(), + function_name: actf_tool_name(call), + arguments: actf_tool_arguments(call), + result: Default::default(), + duration_ms: if step.tools.len() == 1 { + step.metric.env_action_ms.as_f64().map(|value| value as i64) + } else { + None + }, + extra: Some(json!({ + ACTF_EXTENSION_KEY: tool_residual(call)?, + })), + }) + }) + .collect::>>() + }) + .transpose()?; let observation = (!step.observation.is_empty()).then(|| { let results = step .observation @@ -210,7 +244,9 @@ fn attempt_to_storyline( is_copied_context: None, latency_ms: step.metric.llm_infer_ms.as_f64().map(|value| value as i64), ttft_ms: None, - extra: Some(json!({ "_pchronicle_actf_step": step })), + extra: Some(json!({ + ACTF_EXTENSION_KEY: step_residual(step)?, + })), }); } @@ -243,7 +279,7 @@ fn attempt_to_storyline( })), continued_trajectory_ref: None, extra: Some(json!({ - "_pchronicle_actf": { + ACTF_EXTENSION_KEY: { "root": root_metadata, "attempt_id": attempt_id, "attempt": attempt_metadata, @@ -339,24 +375,59 @@ fn synthesize_actf(story: &StorylineDocument) -> Result { } fn synthesize_step(turn: &StorylineTurn) -> Result { + serde_json::from_value(storyline_step_value(turn)?) + .map_err(|error| Error::Other(format!("build ACTF step {}: {error}", turn.id))) +} + +fn storyline_step_value(turn: &StorylineTurn) -> Result { let tools = turn .tool_calls .as_deref() .unwrap_or_default() .iter() - .map(|call| ActfToolCall { - kind: call + .map(|call| { + let metadata = call .extra .as_ref() - .and_then(|extra| extra.get("actf_type")) + .and_then(|extra| extra.get(ACTF_EXTENSION_KEY)) + .and_then(Value::as_object); + let mut tool = metadata + .and_then(|value| value.get("residual")) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + let kind = metadata + .and_then(|value| value.get("kind")) + .and_then(Value::as_str) + .unwrap_or("tool_use"); + tool.insert("type".into(), Value::String(kind.into())); + tool.insert("id".into(), Value::String(call.tool_call_id.clone())); + if metadata + .and_then(|value| value.get("name_present")) + .and_then(Value::as_bool) + .unwrap_or(true) + { + tool.insert("name".into(), Value::String(call.function_name.clone())); + } + match metadata + .and_then(|value| value.get("arguments_key")) .and_then(Value::as_str) - .unwrap_or("tool_use") - .to_string(), - id: call.tool_call_id.clone(), - extra: Map::from_iter([ - ("name".into(), Value::String(call.function_name.clone())), - ("input".into(), call.arguments.clone()), - ]), + .unwrap_or("input") + { + "command" => { + let command = call + .arguments + .get("command") + .cloned() + .unwrap_or_else(|| call.arguments.clone()); + tool.insert("command".into(), command); + } + "none" => {} + _ => { + tool.insert("input".into(), call.arguments.clone()); + } + } + Value::Object(tool) }) .collect::>(); let observations = turn @@ -367,61 +438,58 @@ fn synthesize_step(turn: &StorylineTurn) -> Result { .into_iter() .flatten() .map(|result| { - let tool_use_id = result - .get("tool_use_id") - .or_else(|| result.get("source_call_id")) - .and_then(Value::as_str) - .unwrap_or("") - .to_string(); let mut extra = result.as_object().cloned().unwrap_or_default(); - extra.remove("type"); extra.remove("source_call_id"); - extra - .entry("tool_use_id") - .or_insert(Value::String(tool_use_id)); - ActfObservation { - kind: result - .get("type") - .and_then(Value::as_str) - .unwrap_or("tool_result") - .to_string(), - extra, - } + Value::Object(extra) }) .collect::>(); - let metric = turn - .metrics - .clone() - .and_then(|value| serde_json::from_value::(value).ok()) - .unwrap_or(ActfMetric { - prompt_tokens_len: json!(0), - completion_tokens_len: json!(0), - llm_infer_ms: turn.latency_ms.map_or(Value::Null, |value| json!(value)), - env_action_ms: Value::Null, - stop_reason: Value::Null, - extra: Map::new(), - }); + let metric = turn.metrics.clone().unwrap_or_else(|| { + json!({ + "prompt_tokens_len": 0, + "completion_tokens_len": 0, + "llm_infer_ms": turn.latency_ms.map_or(Value::Null, |value| json!(value)), + "env_action_ms": Value::Null, + "stop_reason": Value::Null, + }) + }); let timestamp = turn .timestamp .clone() .unwrap_or_else(|| "1970-01-01 00:00:00+00:00".into()); - Ok(ActfStep { - step_id: turn.id, - assistant_content: ActfAssistantContent { - content: turn.message.as_str().unwrap_or("").to_string(), - reasoning_content: turn.reasoning_content.clone().unwrap_or_default(), - tool_calls: tools.clone(), - extra: Map::new(), - }, - metric, - system_prompt: String::new(), - user_content: String::new(), - tools, - observation: observations, - started_at: timestamp.clone(), - finished_at: timestamp, - extra: Map::new(), - }) + let metadata = turn + .extra + .as_ref() + .and_then(|extra| extra.get(ACTF_EXTENSION_KEY)) + .and_then(Value::as_object); + let mut assistant = metadata + .and_then(|value| value.get("assistant_content")) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + assistant.insert( + "content".into(), + Value::String(turn.message.as_str().unwrap_or("").to_string()), + ); + assistant.insert( + "reasoning_content".into(), + Value::String(turn.reasoning_content.clone().unwrap_or_default()), + ); + assistant.insert("tool_calls".into(), Value::Array(tools.clone())); + + let mut step = Map::new(); + step.insert("step_id".into(), json!(turn.id)); + step.insert("assistant_content".into(), Value::Object(assistant)); + step.insert("metric".into(), metric); + step.insert("tools".into(), Value::Array(tools)); + step.insert("observation".into(), Value::Array(observations)); + step.insert("started_at".into(), Value::String(timestamp)); + if let Some(residual) = metadata + .and_then(|value| value.get("step")) + .and_then(Value::as_object) + { + merge_residual(&mut step, residual, "step"); + } + Ok(Value::Object(step)) } fn actf_tool_name(call: &ActfToolCall) -> String { @@ -453,23 +521,27 @@ fn actf_observation_call_id(observation: &ActfObservation) -> Option<&str> { fn root_metadata(document: &ActfDocument) -> Result { let mut value = serde_json::to_value(document)?; - value + let object = value .as_object_mut() - .ok_or_else(|| Error::Other("serialized ACTF document must be an object".into()))? - .remove("attempts"); + .ok_or_else(|| Error::Other("serialized ACTF document must be an object".into()))?; + for key in ["task_id", "correct", "attempts"] { + object.remove(key); + } Ok(value) } -fn attempt_metadata(attempt: &ActfAttempt) -> Result { +fn attempt_residual(attempt: &ActfAttempt) -> Result { let mut value = serde_json::to_value(attempt)?; - value + let object = value .as_object_mut() - .ok_or_else(|| Error::Other("serialized ACTF attempt must be an object".into()))? - .remove("trajectory"); + .ok_or_else(|| Error::Other("serialized ACTF attempt must be an object".into()))?; + for key in ["correct", "score", "status", "trajectory"] { + object.remove(key); + } Ok(value) } -fn trajectory_metadata(trajectory: &ActfTrajectory) -> Result { +fn trajectory_residual(trajectory: &ActfTrajectory) -> Result { let mut value = serde_json::to_value(trajectory)?; value .as_object_mut() @@ -478,8 +550,68 @@ fn trajectory_metadata(trajectory: &ActfTrajectory) -> Result { Ok(value) } -fn provenance(story: &StorylineDocument) -> Option<&Map> { - story.extra.as_ref()?.get(ACTF_PROVENANCE_KEY)?.as_object() +fn step_residual(step: &ActfStep) -> Result { + let mut value = serde_json::to_value(step)?; + let object = value + .as_object_mut() + .ok_or_else(|| Error::Other("serialized ACTF step must be an object".into()))?; + let mut assistant = object + .remove("assistant_content") + .and_then(|value| value.as_object().cloned()) + .ok_or_else(|| { + Error::Other("serialized ACTF assistant_content must be an object".into()) + })?; + for key in ["content", "reasoning_content", "tool_calls"] { + assistant.remove(key); + } + for key in ["step_id", "metric", "tools", "observation", "started_at"] { + object.remove(key); + } + let mut residual = Map::new(); + residual.insert("step".into(), Value::Object(object.clone())); + residual.insert("assistant_content".into(), Value::Object(assistant)); + Ok(Value::Object(residual)) +} + +fn tool_residual(call: &ActfToolCall) -> Result { + let name_present = call.extra.contains_key("name"); + let arguments_key = if call.extra.contains_key("input") { + "input" + } else if call.extra.contains_key("command") { + "command" + } else { + "none" + }; + let mut residual = call.extra.clone(); + residual.remove("name"); + residual.remove("input"); + residual.remove("command"); + Ok(json!({ + "kind": call.kind, + "name_present": name_present, + "arguments_key": arguments_key, + "residual": residual, + })) +} + +fn merge_residual(target: &mut Map, residual: &Map, scope: &str) { + for (key, value) in residual { + if target.contains_key(key) { + tracing::warn!( + source_format = "actf", + source_key = %key, + target_key = %key, + scope, + "ACTF residual conflicts with an authoritative Storyline field" + ); + continue; + } + target.insert(key.clone(), value.clone()); + } +} + +fn residual(story: &StorylineDocument) -> Option<&Map> { + story.extra.as_ref()?.get(ACTF_EXTENSION_KEY)?.as_object() } #[cfg(test)] @@ -518,6 +650,49 @@ mod tests { assert_eq!(storyline_to_actf(&story).unwrap(), document); } + #[test] + fn actf_residual_preserves_unknowns_but_storyline_fields_are_authoritative() { + let mut value: Value = serde_json::from_str(FIXTURE).unwrap(); + value["root_unknown"] = Value::Null; + value["attempts"]["1"]["attempt_unknown"] = json!([3, 2, 1]); + value["attempts"]["1"]["trajectory"]["trajectory_unknown"] = json!({"x": 1}); + value["attempts"]["1"]["trajectory"]["steps"][0]["step_unknown"] = Value::Null; + value["attempts"]["1"]["trajectory"]["steps"][0]["assistant_content"] + ["assistant_unknown"] = json!("kept"); + value["attempts"]["1"]["trajectory"]["steps"][0]["tools"][0]["tool_unknown"] = Value::Null; + value["attempts"]["1"]["trajectory"]["steps"][0]["assistant_content"]["tool_calls"][0] + ["tool_unknown"] = Value::Null; + let document: ActfDocument = serde_json::from_value(value).unwrap(); + + let mut story = actf_to_storyline(&document).unwrap(); + assert!(!serde_json::to_string(&story) + .unwrap() + .contains("_pchronicle_")); + story.turns[0].message = json!("changed by Storyline"); + story.turns[0].reasoning_content = Some("new reasoning".into()); + + let recovered = storyline_to_actf(&story).unwrap(); + let recovered = serde_json::to_value(recovered).unwrap(); + let step = &recovered["attempts"]["1"]["trajectory"]["steps"][0]; + assert_eq!(step["assistant_content"]["content"], "changed by Storyline"); + assert_eq!( + step["assistant_content"]["reasoning_content"], + "new reasoning" + ); + assert_eq!(recovered["root_unknown"], Value::Null); + assert_eq!( + recovered["attempts"]["1"]["attempt_unknown"], + json!([3, 2, 1]) + ); + assert_eq!( + recovered["attempts"]["1"]["trajectory"]["trajectory_unknown"], + json!({"x": 1}) + ); + assert_eq!(step["step_unknown"], Value::Null); + assert_eq!(step["assistant_content"]["assistant_unknown"], "kept"); + assert_eq!(step["tools"][0]["tool_unknown"], Value::Null); + } + #[test] fn multiple_attempts_roundtrip_as_multiple_storylines() { let mut document = parse_actf_document(FIXTURE).unwrap(); From 42b317f37c85c840ff62eed317982f0756b1a5bc Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:33:22 +0800 Subject: [PATCH 21/65] fix: make openai residual roundtrips authoritative --- .../persisting-pchronicle/src/convert/actf.rs | 35 +- .../src/formats/openai_corpus.rs | 453 ++++++++++++++++-- 2 files changed, 442 insertions(+), 46 deletions(-) diff --git a/crates/persisting-pchronicle/src/convert/actf.rs b/crates/persisting-pchronicle/src/convert/actf.rs index 74adc0cc..b5a5dc0b 100644 --- a/crates/persisting-pchronicle/src/convert/actf.rs +++ b/crates/persisting-pchronicle/src/convert/actf.rs @@ -482,7 +482,13 @@ fn storyline_step_value(turn: &StorylineTurn) -> Result { step.insert("metric".into(), metric); step.insert("tools".into(), Value::Array(tools)); step.insert("observation".into(), Value::Array(observations)); - step.insert("started_at".into(), Value::String(timestamp)); + let timestamp_style = metadata + .and_then(|value| value.get("started_at_style")) + .and_then(Value::as_str); + step.insert( + "started_at".into(), + Value::String(format_actf_timestamp(×tamp, timestamp_style)?), + ); if let Some(residual) = metadata .and_then(|value| value.get("step")) .and_then(Value::as_object) @@ -570,6 +576,10 @@ fn step_residual(step: &ActfStep) -> Result { let mut residual = Map::new(); residual.insert("step".into(), Value::Object(object.clone())); residual.insert("assistant_content".into(), Value::Object(assistant)); + residual.insert( + "started_at_style".into(), + Value::String(timestamp_style(&step.started_at).into()), + ); Ok(Value::Object(residual)) } @@ -610,6 +620,29 @@ fn merge_residual(target: &mut Map, residual: &Map } } +fn timestamp_style(value: &str) -> &'static str { + if value.contains(' ') { + "space-offset" + } else if value.ends_with('Z') { + "rfc3339-z" + } else { + "rfc3339-offset" + } +} + +fn format_actf_timestamp(value: &str, style: Option<&str>) -> Result { + let timestamp = chrono::DateTime::parse_from_rfc3339(value).map_err(|error| { + Error::Other(format!( + "format ACTF timestamp '{value}' from Storyline: {error}" + )) + })?; + Ok(match style { + Some("space-offset") => timestamp.format("%Y-%m-%d %H:%M:%S%:z").to_string(), + Some("rfc3339-offset") => timestamp.to_rfc3339(), + _ => timestamp.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true), + }) +} + fn residual(story: &StorylineDocument) -> Option<&Map> { story.extra.as_ref()?.get(ACTF_EXTENSION_KEY)?.as_object() } diff --git a/crates/persisting-pchronicle/src/formats/openai_corpus.rs b/crates/persisting-pchronicle/src/formats/openai_corpus.rs index d8125711..98e55794 100644 --- a/crates/persisting-pchronicle/src/formats/openai_corpus.rs +++ b/crates/persisting-pchronicle/src/formats/openai_corpus.rs @@ -2,9 +2,8 @@ //! //! Unlike [`super::openai_msg`], which models one `session_steps.json` //! document, this adapter accepts a top-level array containing rows from many -//! sessions. The original row is retained in Storyline `extra`, so the -//! normalized three-table projection remains queryable without making the -//! reverse conversion depend on that projection. +//! sessions. Unmapped container and row fields are retained as hierarchical +//! residuals; mapped content is always regenerated from Storyline. use std::collections::{HashMap, HashSet}; use std::fs; @@ -18,9 +17,15 @@ use crate::formats::storyline::{ }; use crate::{Error, Result}; -const LOSSLESS_FILE_KEY: &str = "_pchronicle_openai_file"; -const LOSSLESS_RECORD_KEY: &str = "_pchronicle_openai_record"; -const NORMALIZED_CONTEXT_KEY: &str = "_pchronicle_openai_context"; +const OPENAI_EXTENSION_KEY: &str = "persisting.dev/openai-msg/v1"; +const ROW_METRIC_FIELDS: &[&str] = &[ + "reward", + "step_reward", + "is_terminal", + "is_truncated", + "is_session_completed", + "is_trainable", +]; /// One source JSON file reconstructed from lossless OpenAI import metadata. #[derive(Debug, Clone, PartialEq)] @@ -159,7 +164,7 @@ pub fn recover_openai_msg_files( let file = story .extra .as_ref() - .and_then(|extra| extra.get(LOSSLESS_FILE_KEY)) + .and_then(|extra| extra.get(OPENAI_EXTENSION_KEY)) .and_then(Value::as_object) .ok_or_else(|| { Error::Other(format!( @@ -195,18 +200,18 @@ pub fn recover_openai_msg_files( } for turn in &story.turns { + if turn.source == "user" { + continue; + } let extra = turn.extra.as_ref().ok_or_else(|| { Error::Other(format!( "Storyline '{}' step {} has no OpenAI provenance", story.session_id, turn.id )) })?; - let Some(record) = extra.get(LOSSLESS_RECORD_KEY).and_then(Value::as_object) else { - if extra.get(NORMALIZED_CONTEXT_KEY).is_some() { - continue; - } + let Some(record) = extra.get(OPENAI_EXTENSION_KEY).and_then(Value::as_object) else { return Err(Error::Other(format!( - "Storyline '{}' step {} has no lossless OpenAI record", + "Storyline '{}' step {} has no OpenAI residual", story.session_id, turn.id ))); }; @@ -224,10 +229,7 @@ pub fn recover_openai_msg_files( .get("ordinal") .and_then(Value::as_u64) .ok_or_else(|| Error::Other("OpenAI record missing ordinal".into()))?; - let raw = record - .get("value") - .cloned() - .ok_or_else(|| Error::Other("OpenAI record missing value".into()))?; + let raw = recover_record(story, turn, record)?; group.records.push((ordinal, raw)); } } @@ -296,7 +298,7 @@ pub fn is_lossless_openai_storyline(story: &StorylineDocument) -> bool { story .extra .as_ref() - .and_then(|extra| extra.get(LOSSLESS_FILE_KEY)) + .and_then(|extra| extra.get(OPENAI_EXTENSION_KEY)) .and_then(Value::as_object) .is_some() } @@ -373,7 +375,7 @@ fn rows_to_storyline( } } - let output = select_output_message(row).ok_or_else(|| { + let (output, output_location) = select_output_message(row).ok_or_else(|| { Error::Other(format!( "OpenAI corpus {} row {} has no assistant output", relative_path, ordinal @@ -405,13 +407,15 @@ fn rows_to_storyline( .map(str::to_string) .unwrap_or_else(|| format!("step-{step_id}")); let request_messages = row.get("messages").cloned(); - if let Some(message) = last_user_message(request_messages.as_ref()) { + let user_message = last_user_message(request_messages.as_ref()); + let user_turn_id = user_message.as_ref().map(|_| next_turn_id); + if let Some((_, message)) = user_message.as_ref() { turns.push(StorylineTurn { id: next_turn_id, kind: Some("llm.request".into()), timestamp: timestamp.clone(), source: "user".into(), - message, + message: message.clone(), reasoning_content: None, reasoning_effort: None, tool_calls: None, @@ -424,7 +428,8 @@ fn rows_to_storyline( ttft_ms: None, extra: Some(json!({ "call_id": call_id, - NORMALIZED_CONTEXT_KEY: { + OPENAI_EXTENSION_KEY: { + "kind": "request", "openai_step_id": step_id, } })), @@ -454,12 +459,16 @@ fn rows_to_storyline( ttft_ms, extra: Some(json!({ "call_id": call_id, - "request_messages": request_messages, - "_pchronicle_openai_record": { - "relative_path": relative_path, - "ordinal": ordinal, - "value": raw, - } + OPENAI_EXTENSION_KEY: record_residual( + row, + relative_path, + ordinal, + step_id, + user_message.as_ref().map(|(index, _)| *index), + user_turn_id, + output_location, + env_state.as_ref(), + )? })), }); next_turn_id += 1; @@ -487,19 +496,294 @@ fn rows_to_storyline( notes: None, final_metrics, continued_trajectory_ref: None, - extra: Some(json!({ "_pchronicle_openai_file": file_metadata })), + extra: Some(json!({ OPENAI_EXTENSION_KEY: file_metadata })), turns, }) } -fn last_user_message(messages: Option<&Value>) -> Option { +fn last_user_message(messages: Option<&Value>) -> Option<(usize, Value)> { messages? .as_array()? .iter() + .enumerate() .rev() - .find(|message| message.get("role").and_then(Value::as_str) == Some("user")) - .and_then(|message| message.get("content")) + .find(|(_, message)| message.get("role").and_then(Value::as_str) == Some("user")) + .and_then(|(index, message)| message.get("content").cloned().map(|value| (index, value))) +} + +#[allow(clippy::too_many_arguments)] +fn record_residual( + row: &Map, + relative_path: &str, + ordinal: usize, + step_id: i64, + user_message_index: Option, + user_turn_id: Option, + output_location: OutputLocation, + env_state: Option<&Value>, +) -> Result { + let mut residual = row.clone(); + for key in ["session_id", "step_id", "messages", "response"] { + residual.remove(key); + } + + let id_present = residual.remove("id").is_some(); + let model_key = ["agent_model", "llm_model"] + .into_iter() + .find(|key| row.get(*key).and_then(Value::as_str).is_some()) + .map(str::to_string); + if let Some(key) = &model_key { + residual.remove(key); + } + let run_key = ["run_id", "run_bucket", "job_id"] + .into_iter() + .find(|key| { + row.get(*key) + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()) + }) + .map(str::to_string); + if let Some(key) = &run_key { + residual.remove(key); + } + + let metric_fields = ROW_METRIC_FIELDS + .iter() + .filter(|field| row.contains_key(**field)) + .map(|field| Value::String((*field).to_string())) + .collect::>(); + for field in ROW_METRIC_FIELDS { + residual.remove(*field); + } + + let timestamp_from_env = env_state + .and_then(|value| value.get("created_at")) + .and_then(Value::as_str) + .is_some(); + let created_at_kind = if timestamp_from_env { + None + } else { + row.get("created_at").map(|value| { + residual.remove("created_at"); + match value { + Value::String(_) => "string", + Value::Number(number) if number.is_i64() || number.is_u64() => "integer", + Value::Number(_) => "float", + _ => "other", + } + }) + }; + + let mut messages = row.get("messages").cloned(); + if let Some(values) = messages.as_mut().and_then(Value::as_array_mut) { + if let Some(index) = user_message_index { + if let Some(message) = values.get_mut(index).and_then(Value::as_object_mut) { + message.remove("content"); + } + } + if let OutputLocation::Message(index) = output_location { + if let Some(message) = values.get_mut(index).and_then(Value::as_object_mut) { + message.remove("content"); + if parse_tool_calls(message.get("tool_calls")).is_some() { + message.remove("tool_calls"); + } + } + } + } + let mut response = row.get("response").cloned(); + if matches!(output_location, OutputLocation::Response) { + if let Some(message) = response.as_mut().and_then(Value::as_object_mut) { + message.remove("content"); + if parse_tool_calls(message.get("tool_calls")).is_some() { + message.remove("tool_calls"); + } + } + } + + let (output_kind, output_index) = match output_location { + OutputLocation::Response => ("response", None), + OutputLocation::Message(index) => ("message", Some(index)), + }; + Ok(json!({ + "relative_path": relative_path, + "ordinal": ordinal, + "step_id": step_id, + "user_message_index": user_message_index, + "user_turn_id": user_turn_id, + "output_kind": output_kind, + "output_index": output_index, + "id_present": id_present, + "model_key": model_key, + "run_key": run_key, + "metric_fields": metric_fields, + "created_at_kind": created_at_kind, + "messages": messages, + "response": response, + "residual": residual, + })) +} + +fn recover_record( + story: &StorylineDocument, + agent_turn: &StorylineTurn, + metadata: &Map, +) -> Result { + let mut record = metadata + .get("residual") + .and_then(Value::as_object) .cloned() + .ok_or_else(|| Error::Other("OpenAI record residual must be an object".into()))?; + let step_id = metadata + .get("step_id") + .and_then(Value::as_i64) + .ok_or_else(|| Error::Other("OpenAI record residual missing step_id".into()))?; + insert_authoritative( + &mut record, + "session_id", + Value::String(story.session_id.clone()), + "record", + ); + insert_authoritative(&mut record, "step_id", json!(step_id), "record"); + + if metadata + .get("id_present") + .and_then(Value::as_bool) + .unwrap_or(false) + { + let call_id = agent_turn + .extra + .as_ref() + .and_then(|value| value.get("call_id")) + .and_then(Value::as_str) + .unwrap_or_else(|| ""); + insert_authoritative( + &mut record, + "id", + Value::String(call_id.to_string()), + "record", + ); + } + if let Some(key) = metadata.get("model_key").and_then(Value::as_str) { + if let Some(model) = &agent_turn.model_name { + insert_authoritative(&mut record, key, Value::String(model.clone()), "record"); + } + } + if let Some(key) = metadata.get("run_key").and_then(Value::as_str) { + if let Some(run_id) = &story.run_id { + insert_authoritative(&mut record, key, Value::String(run_id.clone()), "record"); + } + } + if let Some(fields) = metadata.get("metric_fields").and_then(Value::as_array) { + for field in fields.iter().filter_map(Value::as_str) { + if let Some(value) = agent_turn + .metrics + .as_ref() + .and_then(|value| value.get(field)) + { + insert_authoritative(&mut record, field, value.clone(), "record"); + } + } + } + if let Some(kind) = metadata.get("created_at_kind").and_then(Value::as_str) { + if let Some(timestamp) = agent_turn.timestamp.as_deref() { + insert_authoritative( + &mut record, + "created_at", + encode_timestamp(timestamp, kind)?, + "record", + ); + } + } + + let user_turn = metadata + .get("user_turn_id") + .and_then(Value::as_i64) + .and_then(|id| story.turns.iter().find(|turn| turn.id == id)); + let output_kind = metadata + .get("output_kind") + .and_then(Value::as_str) + .ok_or_else(|| Error::Other("OpenAI record residual missing output_kind".into()))?; + let output_index = metadata.get("output_index").and_then(Value::as_u64); + + if let Some(mut messages) = metadata.get("messages").filter(|v| !v.is_null()).cloned() { + let values = messages + .as_array_mut() + .ok_or_else(|| Error::Other("OpenAI messages residual must be an array".into()))?; + if let (Some(index), Some(user_turn)) = ( + metadata + .get("user_message_index") + .and_then(Value::as_u64) + .map(|value| value as usize), + user_turn, + ) { + let message = values + .get_mut(index) + .and_then(Value::as_object_mut) + .ok_or_else(|| Error::Other("OpenAI user message residual is invalid".into()))?; + message.insert("content".into(), user_turn.message.clone()); + } + if output_kind == "message" { + let index = output_index + .ok_or_else(|| Error::Other("OpenAI output message index is missing".into()))? + as usize; + let message = values + .get_mut(index) + .and_then(Value::as_object_mut) + .ok_or_else(|| Error::Other("OpenAI output message residual is invalid".into()))?; + apply_output(message, agent_turn)?; + } + insert_authoritative(&mut record, "messages", messages, "record"); + } + + if let Some(mut response) = metadata.get("response").filter(|v| !v.is_null()).cloned() { + if output_kind == "response" { + let message = response + .as_object_mut() + .ok_or_else(|| Error::Other("OpenAI response residual must be an object".into()))?; + apply_output(message, agent_turn)?; + } + insert_authoritative(&mut record, "response", response, "record"); + } + Ok(Value::Object(record)) +} + +fn apply_output(message: &mut Map, turn: &StorylineTurn) -> Result<()> { + message.insert("content".into(), turn.message.clone()); + if let Some(calls) = &turn.tool_calls { + message.insert("tool_calls".into(), encode_tool_calls(calls)?); + } + Ok(()) +} + +fn insert_authoritative(target: &mut Map, key: &str, value: Value, scope: &str) { + if target.contains_key(key) { + tracing::warn!( + source_format = "openai-msg", + source_key = key, + target_key = key, + scope, + "OpenAI residual conflicts with an authoritative Storyline field" + ); + } + target.insert(key.to_string(), value); +} + +fn encode_timestamp(timestamp: &str, kind: &str) -> Result { + if kind == "string" { + return Ok(Value::String(timestamp.to_string())); + } + if kind == "other" { + return Ok(Value::String(timestamp.to_string())); + } + let parsed = chrono::DateTime::parse_from_rfc3339(timestamp).map_err(|error| { + Error::Other(format!("encode OpenAI created_at '{timestamp}': {error}")) + })?; + let millis = parsed.timestamp_millis(); + if kind == "integer" && millis % 1_000 == 0 { + Ok(json!(millis / 1_000)) + } else { + Ok(json!(millis as f64 / 1_000.0)) + } } fn input_files(input: &Path) -> Result> { @@ -595,20 +879,30 @@ fn parsed_env_state(meta: Option<&Value>) -> Option { } } -fn select_output_message(row: &Map) -> Option<&Map> { +#[derive(Debug, Clone, Copy)] +enum OutputLocation { + Response, + Message(usize), +} + +fn select_output_message( + row: &Map, +) -> Option<(&Map, OutputLocation)> { let response = row.get("response").and_then(Value::as_object); if response.is_some_and(message_has_output) { - return response; + return response.map(|value| (value, OutputLocation::Response)); } row.get("messages")? .as_array()? .iter() + .enumerate() .rev() - .filter_map(Value::as_object) - .find(|message| { + .filter_map(|(index, value)| value.as_object().map(|message| (index, message))) + .find(|(_, message)| { message.get("role").and_then(Value::as_str) == Some("assistant") && message_has_output(message) }) + .map(|(index, message)| (message, OutputLocation::Message(index))) } fn message_has_output(message: &Map) -> bool { @@ -650,19 +944,75 @@ fn parse_tool_calls(value: Option<&Value>) -> Option> { } _ => arguments, }; + let mut call_residual = call.clone(); + call_residual.remove("id"); + call_residual.remove("type"); + call_residual.remove("function"); + let mut function_residual = function.clone(); + function_residual.remove("name"); + let raw_arguments = function_residual.remove("arguments"); Some(StorylineToolCall { tool_call_id, function_name, arguments, result: Default::default(), duration_ms: None, - extra: Some(Value::Object(call.clone())), + extra: Some(json!({ + OPENAI_EXTENSION_KEY: { + "kind": "tool_call", + "type": call.get("type"), + "call": call_residual, + "function": function_residual, + "arguments_were_string": raw_arguments.is_some_and(|value| value.is_string()), + } + })), }) }) .collect::>(); (!parsed.is_empty()).then_some(parsed) } +fn encode_tool_calls(calls: &[StorylineToolCall]) -> Result { + calls + .iter() + .map(|call| { + let metadata = call + .extra + .as_ref() + .and_then(|value| value.get(OPENAI_EXTENSION_KEY)) + .and_then(Value::as_object); + let mut output = metadata + .and_then(|value| value.get("call")) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + output.insert("id".into(), Value::String(call.tool_call_id.clone())); + if let Some(kind) = metadata.and_then(|value| value.get("type")) { + output.insert("type".into(), kind.clone()); + } + let mut function = metadata + .and_then(|value| value.get("function")) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + function.insert("name".into(), Value::String(call.function_name.clone())); + let arguments = if metadata + .and_then(|value| value.get("arguments_were_string")) + .and_then(Value::as_bool) + .unwrap_or(false) + { + Value::String(serde_json::to_string(&call.arguments)?) + } else { + call.arguments.clone() + }; + function.insert("arguments".into(), arguments); + output.insert("function".into(), Value::Object(function)); + Ok(Value::Object(output)) + }) + .collect::>>() + .map(Value::Array) +} + fn parse_embedded_tool_call( content: Option<&Value>, step_id: i64, @@ -726,14 +1076,6 @@ fn message_text(content: &Value) -> Option { } fn normalized_metrics(row: &Map, env_state: Option<&Value>) -> Option { - const ROW_FIELDS: &[&str] = &[ - "reward", - "step_reward", - "is_terminal", - "is_truncated", - "is_session_completed", - "is_trainable", - ]; const ENV_FIELDS: &[&str] = &[ "prompt_tokens", "completion_tokens", @@ -747,7 +1089,7 @@ fn normalized_metrics(row: &Map, env_state: Option<&Value>) -> Op "ttft_ms", ]; let mut metrics = Map::new(); - for field in ROW_FIELDS { + for field in ROW_METRIC_FIELDS { if let Some(value) = row.get(*field) { metrics.insert((*field).to_string(), value.clone()); } @@ -854,6 +1196,27 @@ mod tests { assert_eq!(recovered[0].document, input); } + #[test] + fn openai_residual_preserves_unknowns_but_storyline_content_is_authoritative() { + let input = corpus(); + let mut stories = parse_openai_msg_corpus_value(&input, "corpus.json").unwrap(); + assert!(!serde_json::to_string(&stories) + .unwrap() + .contains("_pchronicle_")); + + stories[0].turns[0].message = json!("edited user"); + stories[0].turns[1].message = json!("edited assistant"); + let recovered = recover_openai_msg_files(&stories).unwrap(); + let rows = recovered[0].document.as_array().unwrap(); + let first_session_row = rows.iter().find(|row| row["id"] == "evt-1").unwrap(); + assert_eq!(first_session_row["messages"][1]["content"], "edited user"); + assert_eq!( + first_session_row["messages"][2]["content"], + "edited assistant" + ); + assert_eq!(rows[0]["unknown"], Value::Null); + } + #[test] fn envelope_roundtrip_preserves_root_metadata() { let input = json!({ From fedcc2cd8c1391f5d3d974fa4dabb309bebd1e53 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 17 Aug 2026 23:35:11 +0800 Subject: [PATCH 22/65] test: enforce lossless storyline lance roundtrips --- crates/persisting-pchronicle/Cargo.toml | 4 ++ .../tests/storyline_lance_roundtrip.rs | 64 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 crates/persisting-pchronicle/tests/storyline_lance_roundtrip.rs diff --git a/crates/persisting-pchronicle/Cargo.toml b/crates/persisting-pchronicle/Cargo.toml index 00df5cc5..eeedb285 100644 --- a/crates/persisting-pchronicle/Cargo.toml +++ b/crates/persisting-pchronicle/Cargo.toml @@ -83,3 +83,7 @@ harness = false [[test]] name = "search_integration" required-features = ["search"] + +[[test]] +name = "storyline_lance_roundtrip" +required-features = ["lance-store"] diff --git a/crates/persisting-pchronicle/tests/storyline_lance_roundtrip.rs b/crates/persisting-pchronicle/tests/storyline_lance_roundtrip.rs new file mode 100644 index 00000000..e79fd934 --- /dev/null +++ b/crates/persisting-pchronicle/tests/storyline_lance_roundtrip.rs @@ -0,0 +1,64 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use persisting_pchronicle::convert::{atif_to_storyline, storyline_to_atif}; +use persisting_pchronicle::{ + actf_to_storylines, recover_openai_msg_files, storylines_to_actf, ActfDocument, AtifTrajectory, + OpenaiMsgCorpusReader, StorylineDocument, StorylineLanceStore, +}; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/import_roundtrip") + .join(name) +} + +async fn persist_and_restore(stories: &[StorylineDocument]) -> Result> { + let temporary = tempfile::tempdir()?; + let store = StorylineLanceStore::open(temporary.path()).await?; + store.replace_storylines(stories).await?; + let session_ids = stories + .iter() + .map(|story| story.session_id.clone()) + .collect::>(); + store + .get_storylines_full(&session_ids) + .await? + .into_iter() + .map(|story| story.context("Storyline Lance roundtrip lost a session")) + .collect() +} + +#[tokio::test] +async fn atif_actf_and_openai_are_lossless_through_storyline_lance() -> Result<()> { + let atif_path = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/atif/parallel_tools_14.json"); + let atif_raw = std::fs::read_to_string(&atif_path)?; + let atif_expected: serde_json::Value = serde_json::from_str(&atif_raw)?; + let atif = AtifTrajectory::from_json_str(&atif_raw)?; + let atif_restored = persist_and_restore(&[atif_to_storyline(&atif)?]).await?; + assert_eq!( + serde_json::to_value(storyline_to_atif(&atif_restored[0])?)?, + atif_expected + ); + + let actf_path = fixture("make-doom-for-mips_trimmed.actf.json"); + let actf_raw = std::fs::read_to_string(&actf_path)?; + let actf_expected: serde_json::Value = serde_json::from_str(&actf_raw)?; + let actf = ActfDocument::from_json_str(&actf_raw)?; + let actf_restored = persist_and_restore(&actf_to_storylines(&actf)?).await?; + assert_eq!( + serde_json::to_value(storylines_to_actf(&actf_restored)?)?, + actf_expected + ); + + let openai_path = fixture("cybergym_0729001_trimmed.json"); + let openai_expected: serde_json::Value = serde_json::from_slice(&std::fs::read(&openai_path)?)?; + let openai_stories = OpenaiMsgCorpusReader::open(&openai_path)? + .collect::>>()?; + let openai_restored = persist_and_restore(&openai_stories).await?; + let recovered = recover_openai_msg_files(&openai_restored)?; + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].document, openai_expected); + Ok(()) +} From 2301bfa6ef5c2ea7f1c02926ecf3058f299db72b Mon Sep 17 00:00:00 2001 From: Reiase Date: Tue, 18 Aug 2026 00:02:51 +0800 Subject: [PATCH 23/65] refactor: project gateway agenticmd through storyline --- .../src/engine/actors/story.rs | 6 +- .../src/engine/tests/markdown_stream.rs | 72 ++--- .../src/engine/tests/replay_dedup.rs | 10 +- .../src/projection/dialogue/block.rs | 84 ++++- .../src/projection/dialogue/draft.rs | 10 +- .../src/projection/dialogue/mod.rs | 4 +- .../src/projection/dialogue/tests.rs | 72 ++--- .../src/projection/frontmatter.rs | 140 +++++--- .../src/projection/markdown.rs | 102 +++--- .../src/projection/pipeline.rs | 127 ++++---- .../src/projection/reconcile.rs | 113 ++++--- .../persisting-gateway/src/session/client.rs | 10 +- .../tests/agenticmd_bridge.rs | 159 +++------- .../tests/agenticmd_golden.rs | 94 ++---- .../tests/markdown_trajectory.rs | 163 ++++------ crates/persisting-pchronicle/Cargo.toml | 32 ++ .../src/agenticmd/body.rs | 79 ----- .../src/agenticmd/codec.rs | 27 +- .../src/agenticmd/convert.rs | 298 ++++++++++++------ .../src/agenticmd/frontmatter.rs | 109 ------- .../persisting-pchronicle/src/agenticmd/fs.rs | 120 +++++-- .../src/agenticmd/mapping/fields.rs | 99 ------ .../src/agenticmd/mapping/mod.rs | 236 -------------- .../src/agenticmd/mapping/text.rs | 88 ------ .../src/agenticmd/mod.rs | 35 +- .../src/agenticmd/projection.rs | 62 ++-- .../persisting-pchronicle/src/convert/mod.rs | 16 +- .../persisting-pchronicle/src/formats/mod.rs | 15 - crates/persisting-pchronicle/src/lib.rs | 37 +-- .../src/projection/mod.rs | 2 +- crates/persisting-pchronicle/src/store/mod.rs | 8 - crates/persisting-pchronicle/src/tests.rs | 188 ----------- 32 files changed, 933 insertions(+), 1684 deletions(-) delete mode 100644 crates/persisting-pchronicle/src/agenticmd/body.rs delete mode 100644 crates/persisting-pchronicle/src/agenticmd/frontmatter.rs delete mode 100644 crates/persisting-pchronicle/src/agenticmd/mapping/fields.rs delete mode 100644 crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs delete mode 100644 crates/persisting-pchronicle/src/agenticmd/mapping/text.rs diff --git a/crates/persisting-gateway/src/engine/actors/story.rs b/crates/persisting-gateway/src/engine/actors/story.rs index f73426c7..e2409cc6 100644 --- a/crates/persisting-gateway/src/engine/actors/story.rs +++ b/crates/persisting-gateway/src/engine/actors/story.rs @@ -9,7 +9,7 @@ use pulsing_actor::prelude::*; use super::super::story::{StoryId, TurnMachine}; use super::super::wire::{CaptureAck, DraftPayload, StoryCommand, StoryReply, StoryScope}; -use crate::projection::dialogue::draft_stream_assistant_block; +use crate::projection::dialogue::draft_stream_assistant_turn; use crate::projection::frontmatter::refresh_document_frontmatter; use crate::projection::markdown_pipeline::{LiveMarkdownWriter, MarkdownTarget}; use crate::projection::markdown_policy::should_refresh_frontmatter; @@ -184,9 +184,9 @@ impl StoryActor { .sink .peek_next_seq(scope.route()) .context("draft markdown requires sink peek_next_seq")?; - if let Some(block) = draft_stream_assistant_block(&rec, &draft.assistant_content)? { + if let Some(turn) = draft_stream_assistant_turn(&rec, &draft.assistant_content)? { self.md_writer(&scope) - .write_draft(rec.call_id.as_deref().unwrap_or(""), block)?; + .write_draft(rec.call_id.as_deref().unwrap_or(""), turn)?; } } StoryCommand::Flush | StoryCommand::Snapshot { .. } | StoryCommand::LocalSnapshot => { diff --git a/crates/persisting-gateway/src/engine/tests/markdown_stream.rs b/crates/persisting-gateway/src/engine/tests/markdown_stream.rs index 6e4f198b..47ecb456 100644 --- a/crates/persisting-gateway/src/engine/tests/markdown_stream.rs +++ b/crates/persisting-gateway/src/engine/tests/markdown_stream.rs @@ -4,7 +4,7 @@ use super::fixtures::*; async fn stream_markdown_keeps_user_block_when_assistant_upserts() { use super::support::*; use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::read_agenticmd_blocks_from_file as read_blocks_from_file; + use persisting_pchronicle::parse_agenticmd; let sink = RecordingSink::new(); let dir = tempfile::tempdir().unwrap(); @@ -58,39 +58,25 @@ async fn stream_markdown_keeps_user_block_when_assistant_upserts() { flush_engine(&engine).await; let run_dir = trajectory_run_dir(storage.as_path(), ctx.agent_id(), ctx.route()); let md_path = session_markdown_write_path_for_key(&run_dir, &ctx.route().storage_session_id); - let blocks = read_blocks_from_file(&md_path).unwrap(); + let story = parse_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); let records = sink.drain(); assert_eq!(records.len(), 2); assert_eq!(records[0].seq, 0); assert_eq!(records[1].seq, 1); - assert_eq!(blocks.len(), 2); - assert_eq!(blocks[0].role(), Some("user")); - assert_eq!(blocks[0].body, "hello user"); - assert_eq!( - blocks[0] - .header - .fields - .get("event_seq") - .and_then(|v| v.as_u64()), - Some(0) - ); - assert_eq!(blocks[1].role(), Some("assistant")); - assert_eq!(blocks[1].body, "final assistant"); - assert_eq!( - blocks[1] - .header - .fields - .get("event_seq") - .and_then(|v| v.as_u64()), - Some(1) - ); + assert_eq!(story.turns.len(), 2); + assert_eq!(story.turns[0].source, "user"); + assert_eq!(story.turns[0].message, serde_json::json!("hello user")); + assert_eq!(story.turns[0].id, 0); + assert_eq!(story.turns[1].source, "agent"); + assert_eq!(story.turns[1].message, serde_json::json!("final assistant")); + assert_eq!(story.turns[1].id, 1); } #[tokio::test] async fn draft_markdown_uses_peeked_seq_and_matches_final() { use super::support::*; use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::read_agenticmd_blocks_from_file as read_blocks_from_file; + use persisting_pchronicle::parse_agenticmd; let sink = RecordingSink::new(); let dir = tempfile::tempdir().unwrap(); @@ -129,16 +115,10 @@ async fn draft_markdown_uses_peeked_seq_and_matches_final() { flush_engine(&engine).await; let run_dir = trajectory_run_dir(storage.as_path(), ctx.agent_id(), ctx.route()); let md_path = session_markdown_write_path_for_key(&run_dir, &ctx.route().storage_session_id); - let draft_blocks = read_blocks_from_file(&md_path).unwrap(); - assert_eq!(draft_blocks.len(), 2); - assert_eq!( - draft_blocks[1] - .header - .fields - .get("event_seq") - .and_then(|v| v.as_u64()), - Some(1) - ); + let draft_story = parse_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); + assert_eq!(draft_story.turns.len(), 2); + assert_eq!(draft_story.turns[1].id, 1); + assert_eq!(draft_story.turns[1].message, serde_json::json!("wip")); engine .apply( @@ -157,15 +137,9 @@ async fn draft_markdown_uses_peeked_seq_and_matches_final() { .unwrap(); flush_engine(&engine).await; - let final_blocks = read_blocks_from_file(&md_path).unwrap(); - assert_eq!( - final_blocks[1] - .header - .fields - .get("event_seq") - .and_then(|v| v.as_u64()), - Some(1) - ); + let final_story = parse_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); + assert_eq!(final_story.turns[1].id, 1); + assert_eq!(final_story.turns[1].message, serde_json::json!("done")); assert_eq!(sink.drain()[1].seq, 1); } @@ -173,7 +147,7 @@ async fn draft_markdown_uses_peeked_seq_and_matches_final() { async fn overlapping_calls_preserve_later_user_block_in_markdown() { use super::support::*; use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::read_agenticmd_blocks_from_file as read_blocks_from_file; + use persisting_pchronicle::parse_agenticmd; let sink = RecordingSink::new(); let dir = tempfile::tempdir().unwrap(); @@ -257,9 +231,9 @@ async fn overlapping_calls_preserve_later_user_block_in_markdown() { flush_engine(&engine).await; let run_dir = trajectory_run_dir(storage.as_path(), ctx_a.agent_id(), ctx_a.route()); let md_path = session_markdown_write_path_for_key(&run_dir, &ctx_a.route().storage_session_id); - let blocks = read_blocks_from_file(&md_path).unwrap(); - assert_eq!(blocks.len(), 3); - assert_eq!(blocks[0].body, "req-a"); - assert_eq!(blocks[1].body, "final-a"); - assert_eq!(blocks[2].body, "req-b"); + let story = parse_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); + assert_eq!(story.turns.len(), 3); + assert_eq!(story.turns[0].message, serde_json::json!("req-a")); + assert_eq!(story.turns[1].message, serde_json::json!("final-a")); + assert_eq!(story.turns[2].message, serde_json::json!("req-b")); } diff --git a/crates/persisting-gateway/src/engine/tests/replay_dedup.rs b/crates/persisting-gateway/src/engine/tests/replay_dedup.rs index ab843ce9..04f546d0 100644 --- a/crates/persisting-gateway/src/engine/tests/replay_dedup.rs +++ b/crates/persisting-gateway/src/engine/tests/replay_dedup.rs @@ -17,7 +17,7 @@ fn claude_messages_body(user_lines: &[&str]) -> serde_json::Value { async fn replay_dedup_omits_internal_claude_history_request_from_markdown() { use crate::engine::tests::support::*; use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::read_agenticmd_blocks_from_file as read_blocks_from_file; + use persisting_pchronicle::parse_agenticmd; let sink = RecordingSink::new(); let dir = tempfile::tempdir().unwrap(); @@ -102,8 +102,12 @@ async fn replay_dedup_omits_internal_claude_history_request_from_markdown() { flush_engine(&engine).await; let run_dir = trajectory_run_dir(storage.as_path(), ctx.agent_id(), ctx.route()); let md_path = session_markdown_write_path_for_key(&run_dir, &ctx.route().storage_session_id); - let blocks = read_blocks_from_file(&md_path).unwrap(); - let bodies: Vec<_> = blocks.iter().map(|b| b.body.to_string()).collect(); + let story = parse_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); + let bodies: Vec<_> = story + .turns + .iter() + .map(|turn| turn.message.as_str().unwrap_or_default().to_string()) + .collect(); assert_eq!( bodies, vec![ diff --git a/crates/persisting-gateway/src/projection/dialogue/block.rs b/crates/persisting-gateway/src/projection/dialogue/block.rs index 6dade122..584184a0 100644 --- a/crates/persisting-gateway/src/projection/dialogue/block.rs +++ b/crates/persisting-gateway/src/projection/dialogue/block.rs @@ -1,15 +1,79 @@ -use anyhow::Result; -use persisting_pchronicle::{ - event_record_to_agenticmd_block_with_text, AgenticmdBlock, EventRecord, -}; +use anyhow::{Context, Result}; +use persisting_pchronicle::{EventRecord, StorylineTurn}; +use serde_json::{Map, Value}; use super::fields::role_and_body; -/// Build a pChronicle agenticmd block from a capture record (primary write mapping). -/// -/// Uses capture SSE-aware `visible_*` text, then pChronicle mapping (preserves call_id/seq). -pub fn capture_record_to_agenticmd_block(rec: &EventRecord) -> Result { +/// Project one capture event into the authoritative Storyline turn model. +pub fn capture_record_to_storyline_turn(rec: &EventRecord) -> Result { let (role, body) = role_and_body(rec)?; - let event: EventRecord = rec.clone(); - event_record_to_agenticmd_block_with_text(&event, &role, &body) + let source = match role.as_str() { + "user" => "user", + "assistant" | "agent" => "agent", + _ => "system", + }; + let id = i64::try_from(rec.seq).context("event sequence exceeds Storyline turn id range")?; + let metrics = rec + .payload + .get("body") + .and_then(|body| body.get("usage")) + .or_else(|| rec.payload.get("usage")) + .cloned(); + let model_name = rec + .payload + .get("model") + .and_then(Value::as_str) + .map(str::to_owned); + let ttft_ms = rec.payload.get("ttft_ms").and_then(Value::as_i64); + let latency_ms = rec.payload.get("latency_ms").and_then(Value::as_i64); + + let mut extra = Map::new(); + insert_string(&mut extra, "producer", Some(rec.source.as_str())); + insert_string(&mut extra, "trace_id", rec.trace_id.as_deref()); + insert_string(&mut extra, "parent_uuid", rec.parent_uuid.as_deref()); + insert_string(&mut extra, "subagent_id", rec.subagent_id.as_deref()); + insert_string( + &mut extra, + "parent_agent_id", + rec.parent_agent_id.as_deref(), + ); + for key in [ + "path", + "status", + "draft", + "refs_subagent_ids", + "subagent_trajectories", + "subagent_trajectory", + "spawn_hints", + "spawn_links", + ] { + if let Some(value) = rec.payload.get(key) { + extra.insert(key.into(), value.clone()); + } + } + + Ok(StorylineTurn { + id, + kind: Some(rec.kind.clone()), + timestamp: rec.timestamp.clone(), + source: source.into(), + message: Value::String(body), + reasoning_content: None, + reasoning_effort: None, + tool_calls: None, + observation: None, + metrics, + model_name, + llm_call_count: (source == "agent").then_some(1), + is_copied_context: None, + latency_ms, + ttft_ms, + extra: (!extra.is_empty()).then_some(Value::Object(extra)), + }) +} + +fn insert_string(extra: &mut Map, key: &str, value: Option<&str>) { + if let Some(value) = value.filter(|value| !value.is_empty()) { + extra.insert(key.into(), Value::String(value.into())); + } } diff --git a/crates/persisting-gateway/src/projection/dialogue/draft.rs b/crates/persisting-gateway/src/projection/dialogue/draft.rs index eee56e01..5e446ce1 100644 --- a/crates/persisting-gateway/src/projection/dialogue/draft.rs +++ b/crates/persisting-gateway/src/projection/dialogue/draft.rs @@ -1,16 +1,16 @@ use anyhow::Result; -use persisting_pchronicle::AgenticmdBlock; +use persisting_pchronicle::StorylineTurn; use serde_json::json; -use super::block::capture_record_to_agenticmd_block; +use super::block::capture_record_to_storyline_turn; use super::skip_markdown_block; use crate::record::EventRecord; /// Build a streaming draft assistant block (markdown view only; not written to Lance). -pub fn draft_stream_assistant_block( +pub fn draft_stream_assistant_turn( rec: &EventRecord, assistant_content: &str, -) -> Result> { +) -> Result> { if assistant_content.trim().is_empty() { return Ok(None); } @@ -24,5 +24,5 @@ pub fn draft_stream_assistant_block( if skip_markdown_block(&draft) { return Ok(None); } - Ok(Some(capture_record_to_agenticmd_block(&draft)?)) + Ok(Some(capture_record_to_storyline_turn(&draft)?)) } diff --git a/crates/persisting-gateway/src/projection/dialogue/mod.rs b/crates/persisting-gateway/src/projection/dialogue/mod.rs index 82d88455..349ef4f4 100644 --- a/crates/persisting-gateway/src/projection/dialogue/mod.rs +++ b/crates/persisting-gateway/src/projection/dialogue/mod.rs @@ -12,5 +12,5 @@ mod fields; mod tests; pub use super::markdown_pipeline::skip_markdown_block; -pub use block::capture_record_to_agenticmd_block; -pub use draft::draft_stream_assistant_block; +pub use block::capture_record_to_storyline_turn; +pub use draft::draft_stream_assistant_turn; diff --git a/crates/persisting-gateway/src/projection/dialogue/tests.rs b/crates/persisting-gateway/src/projection/dialogue/tests.rs index 69f45480..207a2b57 100644 --- a/crates/persisting-gateway/src/projection/dialogue/tests.rs +++ b/crates/persisting-gateway/src/projection/dialogue/tests.rs @@ -2,16 +2,11 @@ use serde_json::{json, Value}; use super::*; use crate::config::CaptureLevel; -use crate::record::EventRecordExt; use crate::sink::{ llm_request_record, llm_request_summary_record, llm_response_record, llm_response_record_with_content, }; use crate::Call; -use persisting_pchronicle::agenticmd_block_to_replay_json; -use persisting_pchronicle::{ - encode_agenticmd_block_validated, parse_agenticmd_document_validated as parse_document, -}; fn test_call() -> Call { Call { call_id: "call-test".into(), @@ -50,14 +45,14 @@ fn proxy_nested_body_writes_plain_content() { false, &test_call(), ); - let b1 = capture_record_to_agenticmd_block(&req).unwrap(); - let b2 = capture_record_to_agenticmd_block(&resp).unwrap(); - assert_eq!(b1.body, "你好"); - assert_eq!(b2.body, "你好！"); + let t1 = capture_record_to_storyline_turn(&req).unwrap(); + let t2 = capture_record_to_storyline_turn(&resp).unwrap(); + assert_eq!(t1.message, json!("你好")); + assert_eq!(t2.message, json!("你好！")); } #[test] -fn llm_pair_writes_and_replays() { +fn llm_pair_projects_to_storyline_semantics() { let req = llm_request_record( Some("sess".into()), None, @@ -76,18 +71,20 @@ fn llm_pair_writes_and_replays() { false, &test_call(), ); - let b1 = capture_record_to_agenticmd_block(&req).unwrap(); - let b2 = capture_record_to_agenticmd_block(&resp).unwrap(); - let doc = format!( - "{}{}", - encode_agenticmd_block_validated(&b1).unwrap(), - encode_agenticmd_block_validated(&b2).unwrap(), + let user = capture_record_to_storyline_turn(&req).unwrap(); + let assistant = capture_record_to_storyline_turn(&resp).unwrap(); + assert_eq!(user.source, "user"); + assert_eq!(user.message, json!("你好")); + assert_eq!(assistant.source, "agent"); + assert_eq!(assistant.message, json!("你好！")); + assert_eq!( + assistant.metrics, + Some(json!({ + "prompt_tokens": 12, + "completion_tokens": 18, + "total_tokens": 30 + })) ); - let blocks = parse_document(&doc).unwrap(); - let row: Value = - serde_json::from_str(&agenticmd_block_to_replay_json(&blocks[0]).unwrap()).unwrap(); - assert_eq!(row["source"], "user"); - assert_eq!(row["content"], "你好"); } #[test] @@ -106,8 +103,8 @@ fn summary_request_writes_user_content_not_json() { LEVEL, None, ); - let block = capture_record_to_agenticmd_block(&req).unwrap(); - assert_eq!(block.body, "hi"); + let turn = capture_record_to_storyline_turn(&req).unwrap(); + assert_eq!(turn.message, json!("hi")); } #[test] @@ -122,8 +119,8 @@ fn stream_response_with_assistant_content_writes_plain_text() { &test_call(), LEVEL, ); - let block = capture_record_to_agenticmd_block(&resp).unwrap(); - assert_eq!(block.body, "Hi! How can I help you?"); + let turn = capture_record_to_storyline_turn(&resp).unwrap(); + assert_eq!(turn.message, json!("Hi! How can I help you?")); } #[test] @@ -205,29 +202,24 @@ fn subagent_link_fields_in_markdown_block() { Some("done"), &mut registry, ); - let block = capture_record_to_agenticmd_block(&rec).unwrap(); + let turn = capture_record_to_storyline_turn(&rec).unwrap(); assert_eq!( - block - .header - .fields - .get("subagent_id") + turn.extra + .as_ref() + .and_then(Value::as_object) + .and_then(|extra| extra.get("subagent_id")) .and_then(|v| v.as_str()), Some("abc") ); assert_eq!( - block - .header - .fields - .get("subagent_trajectory") + turn.extra + .as_ref() + .and_then(Value::as_object) + .and_then(|extra| extra.get("subagent_trajectory")) .and_then(|v| v.as_str()), Some("agent-abc.md") ); - assert!(block.body.contains("persisting:subagent-self")); - - let imported = persisting_pchronicle::agenticmd_block_to_event_record(&block).unwrap(); - let visible = imported.visible_assistant_text().unwrap_or_default(); - assert_eq!(visible, "done"); - assert!(!visible.contains("persisting:subagent")); + assert_eq!(turn.message, json!("done")); } #[test] diff --git a/crates/persisting-gateway/src/projection/frontmatter.rs b/crates/persisting-gateway/src/projection/frontmatter.rs index 72ae1618..71a768b3 100644 --- a/crates/persisting-gateway/src/projection/frontmatter.rs +++ b/crates/persisting-gateway/src/projection/frontmatter.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; use super::markdown_trajectory::format_duration_human; use crate::session::client::resolve_client_meta_for_run_dir; @@ -11,16 +13,29 @@ use crate::session::index::{SessionIndexStore, SessionSummary}; use crate::session::snapshots::load_snapshot_turn_counts; use crate::session::storage::{trajectory_run_dir, CaptureRoute}; use persisting_pchronicle::{ - count_agenticmd_role, encode_agenticmd_session_frontmatter, is_subagent_session_storage_key, - rewrite_agenticmd_preamble, + encode_agenticmd, is_subagent_session_storage_key, parse_agenticmd, + rewrite_agenticmd_storyline_metadata, StorylineDocument, }; -pub use persisting_pchronicle::AgenticmdSessionFrontmatter as SessionFrontmatterSummary; +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct SessionFrontmatterSummary { + pub session: String, + pub agent: String, + pub model: Option, + pub provider: Option, + pub started: Option, + pub duration: Option, + pub turns: u64, + pub total_tokens: u64, + pub estimated_cost_usd: Option, + pub subagents: Vec, + pub client: Option, +} /// Serialize YAML frontmatter block (including opening/closing `---`). pub fn format_session_frontmatter(summary: &SessionFrontmatterSummary) -> Result { - encode_agenticmd_session_frontmatter(summary) - .map_err(|e| anyhow::anyhow!("pchronicle agenticmd session frontmatter: {e}")) + encode_agenticmd(&storyline_metadata(summary)) + .map_err(|e| anyhow::anyhow!("pchronicle AgenticMD Storyline metadata: {e}")) } /// Build rollup from markdown blocks + `sessions.json` + run directory siblings. @@ -64,7 +79,12 @@ fn count_user_turns(md_path: &Path) -> Result { if !md_path.is_file() { return Ok(0); } - count_agenticmd_role(md_path, "user") + let story = parse_agenticmd(&std::fs::read_to_string(md_path)?)?; + Ok(story + .turns + .iter() + .filter(|turn| turn.source == "user") + .count() as u64) } fn lookup_session_index( @@ -147,12 +167,55 @@ pub fn refresh_document_frontmatter( md_path, story_user_turn_count, )?; - let header = format_session_frontmatter(&summary)?; - rewrite_agenticmd_preamble(md_path, &header) + let raw = + std::fs::read_to_string(md_path).with_context(|| format!("read {}", md_path.display()))?; + let mut document = + parse_agenticmd(&raw).with_context(|| format!("parse AgenticMD {}", md_path.display()))?; + apply_summary(&mut document, &summary); + rewrite_agenticmd_storyline_metadata(md_path, &document) .with_context(|| format!("refresh frontmatter {}", md_path.display()))?; Ok(summary) } +fn storyline_metadata(summary: &SessionFrontmatterSummary) -> StorylineDocument { + let mut document = StorylineDocument::new(&summary.session, &summary.agent); + apply_summary(&mut document, summary); + document +} + +fn apply_summary(document: &mut StorylineDocument, summary: &SessionFrontmatterSummary) { + document.session_id = summary.session.clone(); + document.agent.id = summary.agent.clone(); + document.agent.name = Some(summary.agent.clone()); + document.agent.model_name = summary.model.clone(); + document.child_session_ids = (!summary.subagents.is_empty()).then(|| summary.subagents.clone()); + + let mut agent_extra = Map::new(); + if let Some(provider) = &summary.provider { + agent_extra.insert("provider".into(), json!(provider)); + } + if let Some(client) = &summary.client { + if let Ok(value) = serde_json::to_value(client) { + agent_extra.insert("client".into(), value); + } + } + document.agent.extra = (!agent_extra.is_empty()).then_some(Value::Object(agent_extra)); + + let mut run_extra = Map::new(); + if let Some(started) = &summary.started { + run_extra.insert("started_at".into(), json!(started)); + } + if let Some(duration) = &summary.duration { + run_extra.insert("duration".into(), json!(duration)); + } + document.extra = (!run_extra.is_empty()).then_some(Value::Object(run_extra)); + document.final_metrics = Some(json!({ + "turn_count": summary.turns, + "total_tokens": summary.total_tokens, + "estimated_cost_usd": summary.estimated_cost_usd, + })); +} + /// Refresh frontmatter for every `{run_dir}/*.md` file. /// /// Uses `.capture/story_snapshots.json` turn counts when `turn_counts` is not provided. @@ -213,34 +276,36 @@ pub fn format_run_summary_line(md_path: &Path, summary: &SessionFrontmatterSumma #[cfg(test)] mod tests { use super::*; - use persisting_pchronicle::{ - encode_agenticmd_block_validated, AgenticmdBlock, AgenticmdHeader, - }; - use std::collections::BTreeMap; - - fn user_block(body: &str) -> String { - let mut fields = BTreeMap::new(); - fields.insert("role".into(), serde_json::json!("user")); - encode_agenticmd_block_validated(&AgenticmdBlock { - header: AgenticmdHeader { - type_name: "dialogue".into(), - length: body.len(), - fields, - }, - body: body.to_string(), - }) - .unwrap() + use persisting_pchronicle::StorylineTurn; + + fn user_document(body: &str) -> String { + let mut story = StorylineDocument::new("run-test", "agent"); + story.turns.push(StorylineTurn { + id: 1, + kind: Some("llm.request".into()), + timestamp: None, + source: "user".into(), + message: serde_json::json!(body), + reasoning_content: None, + reasoning_effort: None, + tool_calls: None, + observation: None, + metrics: None, + model_name: None, + llm_call_count: None, + is_copied_context: None, + latency_ms: None, + ttft_ms: None, + extra: None, + }); + encode_agenticmd(&story).unwrap() } #[test] fn refresh_frontmatter_preserves_blocks() { let dir = tempfile::tempdir().unwrap(); let md = dir.path().join("run-test.md"); - std::fs::write( - &md, - format!("---\nformat: persisting\n---\n\n{}", user_block("hello")), - ) - .unwrap(); + std::fs::write(&md, user_document("hello")).unwrap(); let route = CaptureRoute { root_session: Some("run-test".into()), session_id: "run-test".into(), @@ -259,14 +324,7 @@ mod tests { fn frontmatter_prefers_story_turn_count_over_markdown_blocks() { let dir = tempfile::tempdir().unwrap(); let md = dir.path().join("run-test.md"); - std::fs::write( - &md, - format!( - "---\nformat: persisting\n---\n\n{}", - user_block("only one block in md") - ), - ) - .unwrap(); + std::fs::write(&md, user_document("only one block in md")).unwrap(); let route = CaptureRoute { root_session: Some("run-test".into()), session_id: "run-test".into(), @@ -317,11 +375,7 @@ mod tests { let run_dir = dir.path().join(agent).join(root); std::fs::create_dir_all(&run_dir).unwrap(); let md = run_dir.join(format!("{root}.md")); - std::fs::write( - &md, - format!("---\nformat: persisting\n---\n\n{}", user_block("hello")), - ) - .unwrap(); + std::fs::write(&md, user_document("hello")).unwrap(); let call = crate::Call { call_id: "c1".into(), diff --git a/crates/persisting-gateway/src/projection/markdown.rs b/crates/persisting-gateway/src/projection/markdown.rs index 99391ebd..ee0dc2e8 100644 --- a/crates/persisting-gateway/src/projection/markdown.rs +++ b/crates/persisting-gateway/src/projection/markdown.rs @@ -1,42 +1,57 @@ -//! Session markdown (`{session_id}.md`): live append/upsert + capture preamble. -//! -//! Layout path helpers and tolerant parsing live in `persisting_pchronicle`; this -//! module keeps pipeline-aware IO and client-meta preamble. +//! Session AgenticMD writes through the authoritative Storyline model. use std::path::Path; use anyhow::{Context, Result}; -use persisting_pchronicle::{ - append_agenticmd_blocks, encode_agenticmd_preamble, - upsert_block_by_call_id as chronicle_upsert, AgenticmdBlock, AGENTICMD_BLOCK_LAYOUT, - AGENTICMD_FRONTMATTER_FORMAT, -}; -use serde::Serialize; +use persisting_pchronicle::{upsert_agenticmd_turn, StorylineDocument, StorylineTurn}; +use serde_json::{Map, Value}; -use crate::session::client::{resolve_client_meta_for_run_dir, SessionClientMeta}; +use crate::session::client::resolve_client_meta_for_run_dir; -// --- preamble ---------------------------------------------------------------- - -#[derive(Serialize)] -struct DocumentFrontmatter<'a> { - format: &'static str, - block: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - client: Option<&'a SessionClientMeta>, +/// Insert or replace one Storyline turn in the session's readable AgenticMD view. +pub fn upsert_storyline_turn( + path: &Path, + document: &StorylineDocument, + edit_key: &str, + turn: &StorylineTurn, +) -> Result { + let document = if path.exists() { + document.clone() + } else { + document_with_client_metadata(path, document)? + }; + upsert_agenticmd_turn(path, &document, turn, edit_key) + .with_context(|| format!("upsert {}", path.display())) } -/// Build YAML frontmatter; includes detected client process when available. -pub fn format_document_preamble(client: Option<&SessionClientMeta>) -> Result { - let doc = DocumentFrontmatter { - format: AGENTICMD_FRONTMATTER_FORMAT, - block: AGENTICMD_BLOCK_LAYOUT, - client, - }; - encode_agenticmd_preamble(&doc) - .map_err(|e| anyhow::anyhow!("pchronicle agenticmd preamble: {e}")) +fn document_with_client_metadata( + path: &Path, + document: &StorylineDocument, +) -> Result { + let mut document = document.clone(); + let client = path.parent().and_then(|run_dir| { + run_dir + .parent() + .and_then(|agent_dir| agent_dir.parent()) + .and_then(|storage| resolve_client_meta_for_run_dir(storage, run_dir)) + }); + if let Some(client) = client { + let extra = document + .agent + .extra + .get_or_insert_with(|| Value::Object(Map::new())); + let object = extra + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("Storyline agent.extra must be an object"))?; + object.insert( + "client".into(), + serde_json::to_value(client).context("serialize session client metadata")?, + ); + } + Ok(document) } -/// Human-readable duration for frontmatter (`42s`, `3m12s`, `1h5m`). +/// Human-readable duration for session summaries (`42s`, `3m12s`, `1h5m`). pub(crate) fn format_duration_human(secs: u64) -> String { if secs < 60 { return format!("{secs}s"); @@ -52,32 +67,3 @@ pub(crate) fn format_duration_human(secs: u64) -> String { format!("{hours}h{mins}m") } } - -fn write_agenticmd_blocks(path: &Path, blocks: &[AgenticmdBlock]) -> Result { - let preamble = if !path.exists() - || std::fs::metadata(path) - .map(|m| m.len() == 0) - .unwrap_or(true) - { - let client = path.parent().and_then(|run_dir| { - run_dir - .parent() - .and_then(|agent_dir| agent_dir.parent()) - .and_then(|storage| resolve_client_meta_for_run_dir(storage, run_dir)) - }); - Some(format_document_preamble(client.as_ref())?) - } else { - None - }; - append_agenticmd_blocks(path, blocks, preamble.as_deref()) -} - -/// Replace the block whose header `call_id` and source/legacy role match, or append when missing. -pub fn upsert_block_by_call_id(path: &Path, call_id: &str, block: AgenticmdBlock) -> Result { - // New files: seed capture preamble (with optional client meta) before first upsert. - if !path.exists() { - write_agenticmd_blocks(path, std::slice::from_ref(&block))?; - return Ok(false); - } - chronicle_upsert(path, call_id, block).with_context(|| format!("upsert {}", path.display())) -} diff --git a/crates/persisting-gateway/src/projection/pipeline.rs b/crates/persisting-gateway/src/projection/pipeline.rs index a3156290..0db72d4d 100644 --- a/crates/persisting-gateway/src/projection/pipeline.rs +++ b/crates/persisting-gateway/src/projection/pipeline.rs @@ -3,16 +3,16 @@ //! All paths (live `-f md`, materialize, reconcile) go through [`MarkdownPipeline`]. //! Live session actors hold [`LiveMarkdownWriter`] (pipeline + target path + upsert). -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use anyhow::{Context, Result}; -use persisting_pchronicle::AgenticmdBlock; +use persisting_pchronicle::{StorylineDocument, StorylineTurn}; use serde_json::Value; -use super::dialogue::capture_record_to_agenticmd_block; +use super::dialogue::capture_record_to_storyline_turn; use super::markdown_policy::should_skip_record; -use super::markdown_trajectory::upsert_block_by_call_id; +use super::markdown_trajectory::upsert_storyline_turn; use crate::dialogue_extract::count_visible_user_messages; use crate::record::{EventRecord, EventRecordExt}; use crate::session::storage::{trajectory_run_dir, CaptureRoute}; @@ -41,22 +41,23 @@ impl MarkdownPipeline { self.skipped_call_ids.contains(call_id) } - pub fn try_agenticmd_block(&mut self, rec: &EventRecord) -> Result> { + pub fn try_storyline_turn(&mut self, rec: &EventRecord) -> Result> { if self.should_skip(rec) { return Ok(None); } - Ok(Some(capture_record_to_agenticmd_block(rec)?)) + Ok(Some(capture_record_to_storyline_turn(rec)?)) } - pub fn agenticmd_blocks_from_records(records: &[EventRecord]) -> Result> { + pub fn storyline_turns_from_records(records: &[EventRecord]) -> Result> { let mut pipeline = Self::default(); - let mut blocks = Vec::new(); + let mut turns = Vec::new(); for rec in records { - if let Some(block) = pipeline.try_agenticmd_block(rec)? { - blocks.push(block); + if let Some(mut turn) = pipeline.try_storyline_turn(rec)? { + turn.id = turns.len() as i64 + 1; + turns.push(turn); } } - Ok(blocks) + Ok(turns) } pub fn call_ids_from_records(records: &[EventRecord]) -> BTreeSet { @@ -142,14 +143,25 @@ pub struct LiveMarkdownWriter { target: MarkdownTarget, enabled: bool, pipeline: MarkdownPipeline, + document: StorylineDocument, + turn_ids: HashMap<(String, String), i64>, + next_turn_id: i64, } impl LiveMarkdownWriter { pub fn new(target: MarkdownTarget, enabled: bool) -> Self { + let mut document = StorylineDocument::new( + target.route.storage_session_id.clone(), + target.agent_id.clone(), + ); + document.run_id = target.route.root_session.clone(); Self { target, enabled, pipeline: MarkdownPipeline::default(), + document, + turn_ids: HashMap::new(), + next_turn_id: 1, } } @@ -161,25 +173,38 @@ impl LiveMarkdownWriter { if !self.enabled { return Ok(()); } - let Some(block) = self.pipeline.try_agenticmd_block(rec)? else { + let Some(mut turn) = self.pipeline.try_storyline_turn(rec)? else { return Ok(()); }; let call_id = rec.call_id.as_deref().unwrap_or(""); + turn.id = self.turn_id(call_id, &turn.source); let path = self.target.path(); - upsert_block_by_call_id(&path, call_id, block) + upsert_storyline_turn(&path, &self.document, call_id, &turn) .with_context(|| format!("markdown upsert {}", path.display()))?; Ok(()) } - pub fn write_draft(&mut self, call_id: &str, block: AgenticmdBlock) -> Result<()> { + pub fn write_draft(&mut self, call_id: &str, mut turn: StorylineTurn) -> Result<()> { if !self.enabled || self.pipeline.skips_draft(call_id) { return Ok(()); } + turn.id = self.turn_id(call_id, &turn.source); let path = self.target.path(); - upsert_block_by_call_id(&path, call_id, block) + upsert_storyline_turn(&path, &self.document, call_id, &turn) .with_context(|| format!("markdown draft upsert {}", path.display()))?; Ok(()) } + + fn turn_id(&mut self, edit_key: &str, source: &str) -> i64 { + let key = (edit_key.to_string(), source.to_string()); + if let Some(id) = self.turn_ids.get(&key) { + return *id; + } + let id = self.next_turn_id; + self.next_turn_id = self.next_turn_id.saturating_add(1); + self.turn_ids.insert(key, id); + id + } } /// Stamp request payload fields consumed by [`MarkdownPipeline`]. @@ -201,7 +226,7 @@ mod tests { use crate::session::storage::CaptureRoute; use crate::sink::{llm_request_summary_record, llm_response_record_with_content}; use crate::Call; - use persisting_pchronicle::read_agenticmd_blocks_from_file as read_blocks_from_file; + use persisting_pchronicle::parse_agenticmd; use serde_json::json; const LEVEL: CaptureLevel = CaptureLevel::Dialogue; @@ -352,19 +377,19 @@ mod tests { fn intentional_duplicate_user_text_with_increasing_count_both_kept() { let mut p = MarkdownPipeline::default(); assert!(p - .try_agenticmd_block(&request("c1", "hi", 1, None)) + .try_storyline_turn(&request("c1", "hi", 1, None)) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&response("c1", "Hello")) + .try_storyline_turn(&response("c1", "Hello")) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&request("c2", "hi", 2, None)) + .try_storyline_turn(&request("c2", "hi", 2, None)) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&response("c2", "Hi again")) + .try_storyline_turn(&response("c2", "Hi again")) .unwrap() .is_some()); } @@ -373,19 +398,19 @@ mod tests { fn skips_history_replay_without_new_user_turn() { let mut p = MarkdownPipeline::default(); assert!(p - .try_agenticmd_block(&request("c1", "hi", 1, None)) + .try_storyline_turn(&request("c1", "hi", 1, None)) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&response("c1", "Hello")) + .try_storyline_turn(&response("c1", "Hello")) .unwrap() .is_some()); let replay = request("c3", "hi", 1, None); - assert!(p.try_agenticmd_block(&replay).unwrap().is_none()); + assert!(p.try_storyline_turn(&replay).unwrap().is_none()); assert!(p.skips_draft("c3")); assert!(p - .try_agenticmd_block(&response("c3", "internal")) + .try_storyline_turn(&response("c3", "internal")) .unwrap() .is_none()); } @@ -397,28 +422,28 @@ mod tests { let mut p = MarkdownPipeline::default(); assert!(p - .try_agenticmd_block(&request("c1", "hi", 1, None)) + .try_storyline_turn(&request("c1", "hi", 1, None)) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&response("c1", "Hello")) + .try_storyline_turn(&response("c1", "Hello")) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&request("c2", "review", 2, None)) + .try_storyline_turn(&request("c2", "review", 2, None)) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&response("c2", "running tools")) + .try_storyline_turn(&response("c2", "running tools")) .unwrap() .is_some()); - assert!(p.try_agenticmd_block(&tool_req).unwrap().is_some()); + assert!(p.try_storyline_turn(&tool_req).unwrap().is_some()); assert!(p - .try_agenticmd_block(&response("c-tool", "Let me dig in.")) + .try_storyline_turn(&response("c-tool", "Let me dig in.")) .unwrap() .is_some()); assert!(p - .try_agenticmd_block(&response("c-final", "Full design review.")) + .try_storyline_turn(&response("c-final", "Full design review.")) .unwrap() .is_some()); } @@ -438,8 +463,11 @@ mod tests { request("call-4", "你好", 3, None), response("call-4", "你好！有什么我可以帮你的吗？"), ]; - let blocks = MarkdownPipeline::agenticmd_blocks_from_records(&records).unwrap(); - let bodies: Vec<_> = blocks.iter().map(|b| b.body.clone()).collect(); + let turns = MarkdownPipeline::storyline_turns_from_records(&records).unwrap(); + let bodies: Vec<_> = turns + .iter() + .map(|turn| turn.message.as_str().unwrap_or_default().to_string()) + .collect(); assert_eq!( bodies, vec![ @@ -522,14 +550,14 @@ mod tests { .unwrap(); writer.write_record(&response("c4", "你好！")).unwrap(); - let blocks = read_blocks_from_file(&path).unwrap(); - assert_eq!(blocks.len(), 6); - assert_eq!(blocks[0].body, "hi"); - assert_eq!(blocks[1].body, "Hello"); - assert_eq!(blocks[2].body, "hi"); - assert_eq!(blocks[3].body, "Hi again"); - assert_eq!(blocks[4].body, "你好"); - assert_eq!(blocks[5].body, "你好！"); + let story = parse_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(story.turns.len(), 6); + assert_eq!(story.turns[0].message, json!("hi")); + assert_eq!(story.turns[1].message, json!("Hello")); + assert_eq!(story.turns[2].message, json!("hi")); + assert_eq!(story.turns[3].message, json!("Hi again")); + assert_eq!(story.turns[4].message, json!("你好")); + assert_eq!(story.turns[5].message, json!("你好！")); } #[test] @@ -542,18 +570,11 @@ mod tests { writer.write_record(&request("c1", "hi", 1, None)).unwrap(); writer.write_record(&request("c2", "hi", 1, None)).unwrap(); - let draft_block = capture_record_to_agenticmd_block(&response("c2", "draft text")).unwrap(); - writer.write_draft("c2", draft_block).unwrap(); + let draft_turn = capture_record_to_storyline_turn(&response("c2", "draft text")).unwrap(); + writer.write_draft("c2", draft_turn).unwrap(); - let blocks = read_blocks_from_file(&path).unwrap(); - assert_eq!(blocks.len(), 1); - assert_eq!( - blocks[0] - .header - .fields - .get("call_id") - .and_then(|v| v.as_str()), - Some("c1") - ); + let story = parse_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(story.turns.len(), 1); + assert_eq!(story.turns[0].message, json!("hi")); } } diff --git a/crates/persisting-gateway/src/projection/reconcile.rs b/crates/persisting-gateway/src/projection/reconcile.rs index 76971acd..4ddedb73 100644 --- a/crates/persisting-gateway/src/projection/reconcile.rs +++ b/crates/persisting-gateway/src/projection/reconcile.rs @@ -166,47 +166,53 @@ pub fn write_run_reconcile_report(storage: &Path, report: &RunReconcileReport) - mod tests { use super::*; use crate::config::CaptureLevel; + use crate::projection::dialogue::capture_record_to_storyline_turn; use crate::sink::{llm_request_summary_record, llm_response_record_with_content}; use crate::Call; - use persisting_pchronicle::AgenticmdBlock; - use persisting_pchronicle::{encode_agenticmd_block_validated, AgenticmdHeader}; + use persisting_pchronicle::{upsert_agenticmd_turn, StorylineDocument, StorylineTurn}; use std::collections::BTreeMap as Map; - fn block(call_id: &str, role: &str, body: &str) -> String { - let mut fields = Map::new(); - fields.insert("call_id".into(), serde_json::json!(call_id)); - fields.insert("role".into(), serde_json::json!(role)); - encode_agenticmd_block_validated(&AgenticmdBlock { - header: AgenticmdHeader { - type_name: "dialogue".into(), - length: body.len(), - fields, - }, - body: body.to_string(), - }) - .unwrap() + fn write_turns(path: &Path, turns: &[(&str, &str, &str)]) { + let story = StorylineDocument::new("run-test", "agent"); + for (index, (call_id, role, body)) in turns.iter().enumerate() { + let source = if *role == "assistant" { "agent" } else { *role }; + let turn = StorylineTurn { + id: index as i64 + 1, + kind: Some(if source == "user" { + "llm.request".into() + } else { + "llm.response".into() + }), + timestamp: None, + source: source.into(), + message: serde_json::json!(body), + reasoning_content: None, + reasoning_effort: None, + tool_calls: None, + observation: None, + metrics: None, + model_name: None, + llm_call_count: (source == "agent").then_some(1), + is_copied_context: None, + latency_ms: None, + ttft_ms: None, + extra: None, + }; + upsert_agenticmd_turn(path, &story, &turn, call_id).unwrap(); + } } #[test] fn reconcile_detects_missing_and_extra_call_ids() { let dir = tempfile::tempdir().unwrap(); let md = dir.path().join("run-test.md"); - std::fs::write( - &md, - format!( - "{}\n{}", - block("call-a", "user", "hello"), - block("call-a", "assistant", "hi") - ), - ) - .unwrap(); let call = Call { call_id: "call-a".into(), trace_id: "t".into(), started_at: "2026-01-01T00:00:00Z".into(), }; - let req = llm_request_summary_record( + let mut req = llm_request_summary_record( Some("run-test".into()), Some("agent".into()), "m", @@ -220,7 +226,8 @@ mod tests { CaptureLevel::Dialogue, None, ); - let resp = llm_response_record_with_content( + req.seq = 1; + let mut resp = llm_response_record_with_content( Some("run-test".into()), Some("agent".into()), 200, @@ -230,6 +237,22 @@ mod tests { &call, CaptureLevel::Dialogue, ); + resp.seq = 2; + let story = StorylineDocument::new("run-test", "agent"); + upsert_agenticmd_turn( + &md, + &story, + &capture_record_to_storyline_turn(&req).unwrap(), + "call-a", + ) + .unwrap(); + upsert_agenticmd_turn( + &md, + &story, + &capture_record_to_storyline_turn(&resp).unwrap(), + "call-a", + ) + .unwrap(); let mut extra_call = call; extra_call.call_id = "call-b".into(); let orphan = llm_request_summary_record( @@ -305,15 +328,10 @@ mod tests { fn reconcile_story_projection_aligns_when_md_matches() { let dir = tempfile::tempdir().unwrap(); let md = dir.path().join("run-test.md"); - std::fs::write( + write_turns( &md, - format!( - "{}\n{}", - block("call-a", "user", "hello"), - block("call-a", "assistant", "hi") - ), - ) - .unwrap(); + &[("call-a", "user", "hello"), ("call-a", "assistant", "hi")], + ); let call = Call { call_id: "call-a".into(), @@ -357,16 +375,14 @@ mod tests { fn reconcile_story_detects_extra_md_call_id() { let dir = tempfile::tempdir().unwrap(); let md = dir.path().join("run-test.md"); - std::fs::write( + write_turns( &md, - format!( - "{}\n{}\n{}", - block("call-a", "user", "hello"), - block("call-a", "assistant", "hi"), - block("call-ghost", "user", "phantom") - ), - ) - .unwrap(); + &[ + ("call-a", "user", "hello"), + ("call-a", "assistant", "hi"), + ("call-ghost", "user", "phantom"), + ], + ); let call = Call { call_id: "call-a".into(), @@ -409,15 +425,10 @@ mod tests { let run_dir = dir.path().join("run-test"); std::fs::create_dir_all(&run_dir).unwrap(); let md = run_dir.join("run-test.md"); - std::fs::write( + write_turns( &md, - format!( - "{}\n{}", - block("call-a", "user", "hello"), - block("call-a", "assistant", "hi") - ), - ) - .unwrap(); + &[("call-a", "user", "hello"), ("call-a", "assistant", "hi")], + ); let call = Call { call_id: "call-a".into(), diff --git a/crates/persisting-gateway/src/session/client.rs b/crates/persisting-gateway/src/session/client.rs index f1ccafbe..8c7a02ea 100644 --- a/crates/persisting-gateway/src/session/client.rs +++ b/crates/persisting-gateway/src/session/client.rs @@ -18,7 +18,15 @@ use super::storage::{trajectory_run_dir, CaptureRoute}; /// Sidecar metadata for `capture serve` (no `run_session`). `capture run` uses `run_child.yaml` + markdown frontmatter instead. pub const SESSION_CLIENT_META_FILENAME: &str = "session-meta.yaml"; -pub use persisting_pchronicle::AgenticmdClientMeta as SessionClientMeta; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SessionClientMeta { + pub peer: String, + pub peer_port: u16, + pub pid: u32, + pub command: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub machine_fp: Option, +} fn finalize_client_meta(mut meta: SessionClientMeta, peer: SocketAddr) -> SessionClientMeta { if meta.machine_fp.is_none() { diff --git a/crates/persisting-gateway/tests/agenticmd_bridge.rs b/crates/persisting-gateway/tests/agenticmd_bridge.rs index 0354eb27..5d4404cb 100644 --- a/crates/persisting-gateway/tests/agenticmd_bridge.rs +++ b/crates/persisting-gateway/tests/agenticmd_bridge.rs @@ -1,33 +1,25 @@ -//! Capture ↔ pChronicle bridge: debug-view mapping, materialize, and explicit import. +//! Capture → Storyline → AgenticMD bridge tests. -use persisting_gateway::projection::dialogue::capture_record_to_agenticmd_block; +use persisting_gateway::projection::dialogue::capture_record_to_storyline_turn; use persisting_gateway::projection::markdown_pipeline::MarkdownPipeline; -use persisting_gateway::projection::markdown_trajectory::format_document_preamble; use persisting_gateway::record::EventRecord; use persisting_gateway::sink::{llm_request_record, llm_response_record}; use persisting_gateway::Call; -use persisting_pchronicle::{ - agenticmd_block_to_event_record, agenticmd_blocks_to_event_records, - encode_agenticmd_block_validated, markdown_document_to_event_records, parse_agenticmd_document, - parse_agenticmd_document_validated, write_agenticmd_document, -}; +use persisting_pchronicle::{parse_agenticmd, write_agenticmd_storyline, StorylineDocument}; use serde_json::json; fn fixture() -> &'static str { include_str!("fixtures/agenticmd/demo-run-001.md") } -fn fixture_records(document: &str) -> anyhow::Result> { - agenticmd_blocks_to_event_records(&parse_agenticmd_document_validated(document)?) -} - fn materialize_records(path: &std::path::Path, records: &[EventRecord]) -> anyhow::Result<()> { - let blocks = MarkdownPipeline::agenticmd_blocks_from_records(records)?; - write_agenticmd_document(path, &format_document_preamble(None)?, &blocks) + let turns = MarkdownPipeline::storyline_turns_from_records(records)?; + let mut story = StorylineDocument::new("s1", "gateway"); + story.turns = turns; + write_agenticmd_storyline(path, &story) } -#[test] -fn encoded_document_is_accepted_by_both_parser_surfaces() { +fn pair() -> [EventRecord; 2] { let call = Call { call_id: "c1".into(), trace_id: "t1".into(), @@ -35,134 +27,59 @@ fn encoded_document_is_accepted_by_both_parser_surfaces() { }; let mut req = llm_request_record( Some("s1".into()), - None, + Some("gateway".into()), "m", "/v1/chat/completions", &json!({"messages":[{"role":"user","content":"hi"}]}), ); + req.seq = 1; + req.call_id = Some("c1".into()); req.timestamp = Some("2026-01-01T00:00:00Z".into()); let mut resp = llm_response_record( Some("s1".into()), - None, + Some("gateway".into()), 200, &json!({"choices":[{"message":{"role":"assistant","content":"yo"}}]}), false, &call, ); + resp.seq = 2; resp.call_id = Some("c1".into()); - resp.timestamp = Some("2026-01-01T00:00:00Z".into()); - - let mut out = format_document_preamble(None).unwrap(); - for rec in [req, resp] { - let block = capture_record_to_agenticmd_block(&rec).unwrap(); - out.push_str(&encode_agenticmd_block_validated(&block).unwrap()); - } - - let a = parse_agenticmd_document_validated(&out).unwrap(); - let b = parse_agenticmd_document(&out).unwrap().blocks; - assert_eq!(a.len(), b.len()); - for (i, (x, y)) in a.iter().zip(b.iter()).enumerate() { - assert_eq!(x.body, y.body, "encoded body[{i}]"); - agenticmd_block_to_event_record(x).unwrap_or_else(|e| panic!("block[{i}]: {e:#}")); - } + resp.timestamp = Some("2026-01-01T00:00:01Z".into()); + [req, resp] } #[test] -fn explicit_import_uses_chronicle_parse() { - let records = fixture_records(fixture()).expect("AgenticMD import parse"); - assert_eq!(records.len(), 2); - assert!(records.iter().any(|r| { - r.payload - .get("_agenticmd") - .and_then(|v| v.get("source")) - .and_then(|v| v.as_str()) - == Some("user") - })); -} +fn capture_turns_roundtrip_through_public_storyline_api() { + let [req, resp] = pair(); + let mut story = StorylineDocument::new("s1", "gateway"); + story.turns = vec![ + capture_record_to_storyline_turn(&req).unwrap(), + capture_record_to_storyline_turn(&resp).unwrap(), + ]; -#[test] -fn import_keeps_debug_view_metadata() { - let doc = fixture(); - let via_chronicle = markdown_document_to_event_records(doc).unwrap(); - let records = fixture_records(doc).unwrap(); - assert_eq!(records.len(), 2); - for rec in &records { - assert!( - rec.payload - .get("_agenticmd") - .and_then(|v| v.get("block_fields")) - .is_some(), - "expected _agenticmd.block_fields on seq={}", - rec.seq - ); - } - assert_eq!( - records[1].call_id.as_deref(), - Some("call-demo-1"), - "assistant call_id preserved" - ); - assert_eq!( - records[1].payload["_agenticmd"]["block_fields"]["trace_id"].as_str(), - Some("trace-demo-1") - ); - assert_eq!(via_chronicle, records); + let encoded = persisting_pchronicle::encode_agenticmd(&story).unwrap(); + assert_eq!(parse_agenticmd(&encoded).unwrap(), story); } #[test] -fn capture_record_maps_through_agenticmd_block() { - use persisting_pchronicle::encode_agenticmd_block; - - let mut req = llm_request_record( - Some("s1".into()), - None, - "m", - "/v1/chat/completions", - &json!({"messages":[{"role":"user","content":"hi"}]}), - ); - req.seq = 7; - req.call_id = Some("c7".into()); - let block = capture_record_to_agenticmd_block(&req).unwrap(); - assert_eq!(block.header.type_name, "markdown"); - assert_eq!(block.role(), Some("user")); - let wire = encode_agenticmd_block(&block).unwrap(); - assert!(wire.contains("\"event_seq\":7") || wire.contains("\"event_seq\": 7")); - let back = agenticmd_block_to_event_record(&block).unwrap(); - assert_eq!(back.seq, 7); - assert_eq!(back.call_id.as_deref(), Some("c7")); +fn legacy_fixture_still_imports_as_storyline() { + let story = parse_agenticmd(fixture()).expect("legacy AgenticMD import parse"); + assert_eq!(story.turns.len(), 2); + assert_eq!(story.turns[0].source, "user"); + assert_eq!(story.turns[0].message, json!("你好")); + assert_eq!(story.turns[1].source, "agent"); } #[test] -fn materialize_writes_chronicle_parseable_markdown() { - let call = Call { - call_id: "c1".into(), - trace_id: "t1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let mut req = llm_request_record( - Some("s1".into()), - None, - "m", - "/v1/chat/completions", - &json!({"messages":[{"role":"user","content":"hi"}]}), - ); - req.timestamp = Some("2026-01-01T00:00:00Z".into()); - let mut resp = llm_response_record( - Some("s1".into()), - None, - 200, - &json!({"choices":[{"message":{"role":"assistant","content":"yo"}}]}), - false, - &call, - ); - resp.call_id = Some("c1".into()); - resp.timestamp = Some("2026-01-01T00:00:00Z".into()); - +fn materialize_writes_storyline_parseable_markdown() { + let records = pair(); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("sess.md"); - materialize_records(&path, &[req, resp]).unwrap(); - let text = std::fs::read_to_string(&path).unwrap(); - let blocks = parse_agenticmd_document_validated(&text).unwrap(); - assert_eq!(blocks.len(), 2); - let chronicle = parse_agenticmd_document(&text).unwrap(); - assert_eq!(chronicle.blocks.len(), 2); + materialize_records(&path, &records).unwrap(); + let story = parse_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(story.session_id, "s1"); + assert_eq!(story.turns.len(), 2); + assert_eq!(story.turns[0].message, json!("hi")); + assert_eq!(story.turns[1].message, json!("yo")); } diff --git a/crates/persisting-gateway/tests/agenticmd_golden.rs b/crates/persisting-gateway/tests/agenticmd_golden.rs index 5a8bd4fd..63838bee 100644 --- a/crates/persisting-gateway/tests/agenticmd_golden.rs +++ b/crates/persisting-gateway/tests/agenticmd_golden.rs @@ -1,101 +1,63 @@ -//! Golden AgenticMD document built through the production encoder. +//! Golden AgenticMD semantics through the public Storyline API. -use persisting_gateway::projection::dialogue::capture_record_to_agenticmd_block; -use persisting_gateway::record::EventRecordExt; +use persisting_gateway::projection::dialogue::capture_record_to_storyline_turn; use persisting_gateway::sink::{llm_request_record, llm_response_record}; use persisting_gateway::Call; -use persisting_pchronicle::{ - agenticmd_block_to_event_record, encode_agenticmd_block_validated, - encode_agenticmd_session_frontmatter, parse_agenticmd_document_validated as parse_document, - AgenticmdSessionFrontmatter, -}; +use persisting_pchronicle::{encode_agenticmd, parse_agenticmd, StorylineDocument}; use serde_json::json; -fn demo_call() -> Call { - Call { +fn demo_storyline() -> StorylineDocument { + let call = Call { call_id: "call-demo-1".into(), trace_id: "trace-demo-1".into(), started_at: "2026-01-01T00:00:00Z".into(), - } -} - -const DEMO_TIMESTAMP: &str = "2026-01-01T00:00:00Z"; - -fn build_demo_document() -> String { + }; let mut req = llm_request_record( Some("demo-run-001".into()), - None, + Some("demo-agent".into()), "deepseek-chat", "/v1/chat/completions", &json!({"messages":[{"role":"user","content":"你好"}]}), ); req.seq = 1; req.call_id = Some("call-demo-1".into()); - req.timestamp = Some(DEMO_TIMESTAMP.into()); + req.timestamp = Some("2026-01-01T00:00:00Z".into()); let mut resp = llm_response_record( Some("demo-run-001".into()), - None, + Some("demo-agent".into()), 200, &json!({ "choices":[{"message":{"role":"assistant","content":"你好！有什么可以帮你的？"}}], "usage":{"prompt_tokens":12,"completion_tokens":18,"total_tokens":30} }), false, - &demo_call(), + &call, ); - resp.call_id = Some("call-demo-1".into()); resp.seq = 2; resp.timestamp = Some("2026-01-01T00:00:01Z".into()); - let mut out = encode_agenticmd_session_frontmatter(&AgenticmdSessionFrontmatter { - session: "demo-run-001".into(), - agent: "demo-agent".into(), - turns: 1, - ..Default::default() - }) - .unwrap(); - for rec in [req, resp] { - let block = capture_record_to_agenticmd_block(&rec).unwrap(); - out.push_str(&encode_agenticmd_block_validated(&block).unwrap()); - } - // A block keeps a blank separator for append; a closed document ends in - // exactly one newline so the checked-in golden has no trailing blank line. - debug_assert!(out.ends_with("\n\n")); - out.pop(); - out + let mut story = StorylineDocument::new("demo-run-001", "demo-agent"); + story.turns = vec![ + capture_record_to_storyline_turn(&req).unwrap(), + capture_record_to_storyline_turn(&resp).unwrap(), + ]; + story } #[test] -fn demo_run_001_matches_golden_fixture() { - let built = build_demo_document(); - if std::env::var("WRITE_AGENTICMD_GOLDEN").is_ok() { - let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/agenticmd/demo-run-001.md"); - std::fs::write(&fixture, &built).unwrap(); - } - let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/agenticmd/demo-run-001.md"); - let golden = std::fs::read_to_string(&fixture) - .unwrap_or_else(|e| panic!("read {}: {e}", fixture.display())); - assert_eq!( - built, golden, - "regenerate: WRITE_AGENTICMD_GOLDEN=1 cargo test -p persisting-gateway --test agenticmd_golden demo_run_001_matches_golden_fixture" - ); +fn generated_agenticmd_preserves_golden_storyline_semantics() { + let story = demo_storyline(); + let encoded = encode_agenticmd(&story).unwrap(); + assert_eq!(parse_agenticmd(&encoded).unwrap(), story); } #[test] -fn demo_blocks_have_no_version_field_and_strip_subagent_footer_on_import() { - let built = build_demo_document(); - let blocks = parse_document(&built).unwrap(); - assert!(!blocks[0].header.fields.contains_key("v")); - - let mut block = blocks[1].clone(); - block - .body - .push_str("\n\n"); - block.header.length = block.body.len(); - let rec = agenticmd_block_to_event_record(&block).unwrap(); - let content = rec.visible_assistant_text().unwrap_or_default(); - assert!(!content.contains("persisting:subagent")); - assert!(content.contains("你好")); +fn checked_in_legacy_golden_remains_readable() { + let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/agenticmd/demo-run-001.md"); + let story = parse_agenticmd(&std::fs::read_to_string(&fixture).unwrap()).unwrap(); + assert_eq!(story.session_id, "demo-run-001"); + assert_eq!(story.turns.len(), 2); + assert_eq!(story.turns[0].message, json!("你好")); + assert_eq!(story.turns[1].message, json!("你好！有什么可以帮你的？")); } diff --git a/crates/persisting-gateway/tests/markdown_trajectory.rs b/crates/persisting-gateway/tests/markdown_trajectory.rs index 6c11ab6b..e7c7b6d7 100644 --- a/crates/persisting-gateway/tests/markdown_trajectory.rs +++ b/crates/persisting-gateway/tests/markdown_trajectory.rs @@ -1,36 +1,41 @@ -//! Capture-owned client-meta preamble and live upsert seed. +//! Capture-owned metadata and semantic live upsert tests. -use std::collections::BTreeMap; - -use persisting_gateway::projection::markdown_trajectory::{ - format_document_preamble, upsert_block_by_call_id, -}; +use persisting_gateway::projection::markdown_trajectory::upsert_storyline_turn; use persisting_gateway::session::client::{ write_session_client_meta, SessionClientMeta, SESSION_CLIENT_META_FILENAME, }; -use persisting_pchronicle::{ - agenticmd_body_byte_offset, encode_agenticmd_block_validated, - parse_agenticmd_document_validated as parse_document, read_agenticmd_blocks_from_file, - AgenticmdBlock, AgenticmdHeader, -}; +use persisting_pchronicle::{parse_agenticmd, StorylineDocument, StorylineTurn}; +use serde_json::json; -fn block_with_call(call_id: &str, role: &str, body: &str) -> AgenticmdBlock { - let mut fields = BTreeMap::new(); - fields.insert("role".into(), serde_json::json!(role)); - fields.insert("kind".into(), serde_json::json!("llm.response.stream")); - fields.insert("call_id".into(), serde_json::json!(call_id)); - AgenticmdBlock { - header: AgenticmdHeader { - type_name: "markdown".into(), - length: body.len(), - fields, - }, - body: body.into(), +fn turn(id: i64, source: &str, body: &str, draft: bool) -> StorylineTurn { + StorylineTurn { + id, + kind: Some(if draft { + "llm.response.stream".into() + } else if source == "user" { + "llm.request".into() + } else { + "llm.response".into() + }), + timestamp: None, + source: source.into(), + message: json!(body), + reasoning_content: None, + reasoning_effort: None, + tool_calls: None, + observation: None, + metrics: None, + model_name: None, + llm_call_count: (source == "agent").then_some(1), + is_copied_context: None, + latency_ms: None, + ttft_ms: None, + extra: None, } } #[test] -fn preamble_includes_session_client_meta() { +fn new_document_includes_session_client_metadata() { let dir = tempfile::tempdir().unwrap(); let session_dir = dir.path().join("demo-agent").join("sess-1"); std::fs::create_dir_all(&session_dir).unwrap(); @@ -46,95 +51,45 @@ fn preamble_includes_session_client_meta() { ) .unwrap(); - let md_path = session_dir.join("sess-1.md"); - upsert_block_by_call_id( - &md_path, - "call-1", - block_with_call("call-1", "assistant", "hi"), - ) - .unwrap(); + let path = session_dir.join("sess-1.md"); + let story = StorylineDocument::new("sess-1", "demo-agent"); + upsert_storyline_turn(&path, &story, "call-1", &turn(1, "agent", "hi", false)).unwrap(); - let text = std::fs::read_to_string(&md_path).unwrap(); - assert!(text.contains("client:")); - assert!(text.contains("peer_port: 54321")); - assert!(text.contains("claude --model deepseek")); - - let blocks = parse_document(&text).unwrap(); - assert_eq!(blocks.len(), 1); -} - -#[test] -fn parse_document_with_client_frontmatter() { - let preamble = format_document_preamble(Some(&SessionClientMeta { - peer: "127.0.0.1:40000".into(), - peer_port: 40000, - pid: 42, - command: "python3 agent.py".into(), - machine_fp: None, - })) - .unwrap(); - let mut fields = BTreeMap::new(); - fields.insert("role".into(), serde_json::json!("user")); - fields.insert("kind".into(), serde_json::json!("llm.request")); - let block = encode_agenticmd_block_validated(&AgenticmdBlock { - header: AgenticmdHeader { - type_name: "markdown".into(), - length: 2, - fields, - }, - body: "hi".into(), - }) - .unwrap(); - let doc = format!("{preamble}{block}"); - let blocks = parse_document(&doc).unwrap(); - assert_eq!(blocks.len(), 1); - assert_eq!(blocks[0].body, "hi"); + let parsed = parse_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(parsed.turns[0].message, json!("hi")); + assert_eq!( + parsed.agent.extra.as_ref().unwrap()["client"]["peer_port"], + 54321 + ); + assert_eq!( + parsed.agent.extra.as_ref().unwrap()["client"]["command"], + "claude --model deepseek" + ); } #[test] -fn upsert_new_file_seeds_capture_preamble() { +fn live_upsert_replaces_draft_by_edit_key() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("sess.md"); - assert!(!upsert_block_by_call_id( + let story = StorylineDocument::new("sess-1", "demo-agent"); + assert!( + !upsert_storyline_turn(&path, &story, "call-1", &turn(1, "agent", "draft", true),).unwrap() + ); + assert!(upsert_storyline_turn( &path, + &story, "call-1", - block_with_call("call-1", "assistant", "hello"), + &turn(1, "agent", "complete", false), ) .unwrap()); - let raw = std::fs::read_to_string(&path).unwrap(); - assert!(raw.starts_with("---\n")); - assert!(raw.contains("persisting")); - assert!(raw.contains("persisting:block:{speaker}")); - let blocks = read_agenticmd_blocks_from_file(&path).unwrap(); - assert_eq!(blocks.len(), 1); - assert_eq!(blocks[0].body, "hello"); -} -#[test] -fn preamble_roundtrips_through_body_offset() { - let preamble = format_document_preamble(None).unwrap(); - assert!(preamble.starts_with("---\n")); - assert!(preamble.contains("format: persisting")); - assert!(preamble.contains("block:")); - let start = agenticmd_body_byte_offset(&preamble).unwrap(); - assert!( - preamble[start..].trim().is_empty(), - "body after document preamble should be blank, got {:?}", - &preamble[start..] - ); -} - -#[test] -fn preamble_embeds_nested_client() { - let preamble = format_document_preamble(Some(&SessionClientMeta { - peer: "127.0.0.1:1".into(), - peer_port: 1, - pid: 2, - command: "demo".into(), - machine_fp: None, - })) - .unwrap(); - assert!(preamble.contains("client:")); - assert!(preamble.contains("peer_port: 1")); - assert!(preamble.contains("demo")); + let parsed = parse_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(parsed.turns.len(), 1); + assert_eq!(parsed.turns[0].message, json!("complete")); + assert_eq!(parsed.turns[0].kind.as_deref(), Some("llm.response")); + assert!(parsed.turns[0] + .extra + .as_ref() + .and_then(|extra| extra.get("call_id")) + .is_none()); } diff --git a/crates/persisting-pchronicle/Cargo.toml b/crates/persisting-pchronicle/Cargo.toml index eeedb285..fa3802a5 100644 --- a/crates/persisting-pchronicle/Cargo.toml +++ b/crates/persisting-pchronicle/Cargo.toml @@ -87,3 +87,35 @@ required-features = ["search"] [[test]] name = "storyline_lance_roundtrip" required-features = ["lance-store"] + +[[test]] +name = "atif_lance_corpus" +required-features = ["lance-store"] + +[[test]] +name = "capture_fixture_corpus" +required-features = ["lance-store"] + +[[test]] +name = "direct_file_query" +required-features = ["lance-store"] + +[[test]] +name = "import_roundtrip_fixtures" +required-features = ["lance-store"] + +[[test]] +name = "langfuse_backend_faults" +required-features = ["lance-store"] + +[[test]] +name = "production_scale" +required-features = ["lance-store"] + +[[test]] +name = "query_engine" +required-features = ["lance-store"] + +[[test]] +name = "s3_storage" +required-features = ["s3-store"] diff --git a/crates/persisting-pchronicle/src/agenticmd/body.rs b/crates/persisting-pchronicle/src/agenticmd/body.rs deleted file mode 100644 index 12a180cb..00000000 --- a/crates/persisting-pchronicle/src/agenticmd/body.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Agenticmd block body helpers (subagent footers). - -/// Strip `` footer lines from a block body. -/// -/// Footers are human-readable only; they must not round-trip into event message fields. -pub fn strip_subagent_footer_from_body(body: &str) -> String { - let mut lines: Vec<&str> = Vec::new(); - for line in body.lines() { - if is_subagent_footer_line(line) { - continue; - } - lines.push(line); - } - lines.join("\n").trim_end().to_string() -} - -/// True when `line` is a standalone HTML comment footer (after trim). -pub fn is_subagent_footer_line(line: &str) -> bool { - let t = line.trim(); - t.starts_with("") -} - -/// Append visible subagent ref footer for markdown trajectory readers. -pub fn append_subagent_refs_footer(body: &str, payload: &serde_json::Value) -> String { - let mut parts = vec![body.to_string()]; - if let Some(traj) = payload.get("subagent_trajectory").and_then(|v| v.as_str()) { - parts.push(format!("")); - } - if let Some(paths) = payload - .get("subagent_trajectories") - .and_then(|v| v.as_array()) - { - let refs: Vec<_> = paths.iter().filter_map(|p| p.as_str()).collect(); - if !refs.is_empty() { - parts.push(format!( - "", - refs.join(" ") - )); - } - } - if parts.len() == 1 { - return body.to_string(); - } - format!("{}\n", parts.join("\n")) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn strips_subagent_self_and_refs_lines() { - let body = "hello\n\n\n"; - assert_eq!(strip_subagent_footer_from_body(body), "hello"); - } - - #[test] - fn keeps_inline_text_with_similar_substring() { - let body = "see in prose"; - assert_eq!( - strip_subagent_footer_from_body(body), - "see in prose" - ); - } - - #[test] - fn append_footer_from_payload() { - let out = append_subagent_refs_footer( - "done", - &json!({ - "subagent_trajectory": "agent-abc.md", - "subagent_trajectories": ["a.md", "b.md"], - }), - ); - assert!(out.contains("persisting:subagent-self agent-abc.md")); - assert!(out.contains("persisting:subagent-refs a.md b.md")); - } -} diff --git a/crates/persisting-pchronicle/src/agenticmd/codec.rs b/crates/persisting-pchronicle/src/agenticmd/codec.rs index c9e2cfc3..524bb94b 100644 --- a/crates/persisting-pchronicle/src/agenticmd/codec.rs +++ b/crates/persisting-pchronicle/src/agenticmd/codec.rs @@ -77,10 +77,6 @@ impl AgenticmdBlock { .iter() .find_map(|key| self.header.fields.get(*key).and_then(|v| v.as_i64())) } - - pub fn kind(&self) -> Option<&str> { - self.header.fields.get("kind").and_then(|v| v.as_str()) - } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -170,34 +166,13 @@ pub fn agenticmd_body_byte_offset(input: &str) -> Result { /// Encode a YAML frontmatter fence (`---\n…\n---\n\n`) from any serializable mapping. /// -/// Used by capture for live document / session-rollup preambles (nested `client`, etc.). -/// Distinct from [`encode_agenticmd_document`], which emits a flat string frontmatter -/// suitable for hub interchange. +/// Storyline metadata encoding is layered on top by `agenticmd::convert`. pub fn encode_agenticmd_preamble(frontmatter: &T) -> Result { let yaml = serde_yaml::to_string(frontmatter) .map_err(|e| Error::Other(format!("agenticmd frontmatter yaml: {e}")))?; Ok(format!("---\n{yaml}---\n\n")) } -pub fn encode_agenticmd_document(doc: &AgenticmdDocument) -> Result { - let mut frontmatter = doc.frontmatter.clone(); - frontmatter.insert( - "format".into(), - Value::String(doc.frontmatter_format.clone()), - ); - if let Some(session) = &doc.session_id { - frontmatter.insert("session_id".into(), Value::String(session.clone())); - } - if let Some(agent) = &doc.agent_id { - frontmatter.insert("agent_id".into(), Value::String(agent.clone())); - } - let mut out = encode_agenticmd_preamble(&frontmatter)?; - for block in &doc.blocks { - out.push_str(&encode_agenticmd_block(block)?); - } - Ok(out) -} - /// Encode a single agenticmd / capture TLV block (comment header + body). /// /// Normative on-disk layout shared with `persisting-gateway` live write paths. diff --git a/crates/persisting-pchronicle/src/agenticmd/convert.rs b/crates/persisting-pchronicle/src/agenticmd/convert.rs index 04f3554e..bf987cbe 100644 --- a/crates/persisting-pchronicle/src/agenticmd/convert.rs +++ b/crates/persisting-pchronicle/src/agenticmd/convert.rs @@ -5,32 +5,151 @@ use std::collections::BTreeMap; -use serde_json::{json, Map, Value}; +use serde_json::{json, Value}; -use crate::convert::message_text; use crate::formats::storyline::{StorylineAgent, StorylineDocument, StorylineTurn}; -use crate::Result; +use crate::{DocumentFormat, Error, Result}; use super::codec::{ - AgenticmdBlock, AgenticmdDocument, AgenticmdHeader, AGENTICMD_FORMAT_NAME, - AGENTICMD_FRONTMATTER_FORMAT, + encode_agenticmd_block, encode_agenticmd_preamble, parse_agenticmd_document, AgenticmdBlock, + AgenticmdDocument, AgenticmdHeader, AGENTICMD_FRONTMATTER_FORMAT, }; -/// Header field names preserved via `turn.extra` for hub round-trips. -const EXTRA_CORRELATION_KEYS: &[&str] = &[ - "call_id", - "event_seq", - "step_id", - "producer", - "seq", - "turn", - "trace_id", - "parent_uuid", - "draft", - "source", -]; - -pub fn agenticmd_to_storyline(doc: &AgenticmdDocument) -> Result { +const STORYLINE_METADATA_KEY: &str = "storyline"; +const MESSAGE_ENCODING_KEY: &str = "message_encoding"; + +/// Parse AgenticMD into its authoritative Storyline model. +pub fn parse_agenticmd(input: &str) -> Result { + let document = parse_agenticmd_document(input)?; + let Some(metadata) = document.frontmatter.get(STORYLINE_METADATA_KEY) else { + return agenticmd_to_storyline(&document); + }; + let mut story = metadata + .as_object() + .cloned() + .ok_or_else(|| Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + path: None, + location: Some("frontmatter.storyline".into()), + message: "expected an object".into(), + })?; + let turns = document + .blocks + .iter() + .enumerate() + .map(|(index, block)| { + let mut turn = block + .header + .fields + .get(STORYLINE_METADATA_KEY) + .and_then(Value::as_object) + .cloned() + .ok_or_else(|| Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + path: None, + location: Some(format!("block[{index}].storyline")), + message: "expected an object".into(), + })?; + let message = match block + .header + .fields + .get(MESSAGE_ENCODING_KEY) + .and_then(Value::as_str) + { + Some("json") => { + serde_json::from_str(&block.body).map_err(|error| Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + path: None, + location: Some(format!("block[{index}].body")), + message: error.to_string(), + })? + } + _ => Value::String(block.body.clone()), + }; + turn.insert("msg".into(), message); + serde_json::from_value::(Value::Object(turn)).map_err(|error| { + Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + path: None, + location: Some(format!("block[{index}].storyline")), + message: error.to_string(), + } + }) + }) + .collect::>>()?; + story.insert("turns".into(), serde_json::to_value(&turns)?); + let document = + serde_json::from_value::(Value::Object(story)).map_err(|error| { + Error::InvalidDocument { + format: DocumentFormat::AgenticMd, + path: None, + location: Some("frontmatter.storyline".into()), + message: error.to_string(), + } + })?; + document.validate()?; + Ok(document) +} + +/// Encode a Storyline as its human-readable AgenticMD representation. +pub fn encode_agenticmd(story: &StorylineDocument) -> Result { + story.validate()?; + let mut output = encode_storyline_preamble(story)?; + for turn in &story.turns { + output.push_str(&encode_agenticmd_block(&storyline_turn_block(turn, None)?)?); + } + Ok(output) +} + +pub(super) fn encode_storyline_preamble(story: &StorylineDocument) -> Result { + let mut metadata = serde_json::to_value(story)? + .as_object() + .cloned() + .ok_or_else(|| Error::Other("serialized Storyline must be an object".into()))?; + metadata.remove("turns"); + let frontmatter: BTreeMap = BTreeMap::from([ + ( + "format".into(), + Value::String(AGENTICMD_FRONTMATTER_FORMAT.into()), + ), + (STORYLINE_METADATA_KEY.into(), Value::Object(metadata)), + ]); + encode_agenticmd_preamble(&frontmatter) +} + +pub(super) fn storyline_turn_block( + turn: &StorylineTurn, + edit_key: Option<&str>, +) -> Result { + let mut turn_metadata = serde_json::to_value(turn)? + .as_object() + .cloned() + .ok_or_else(|| Error::Other("serialized Storyline turn must be an object".into()))?; + turn_metadata.remove("msg"); + let (body, encoding, type_name) = match &turn.message { + Value::String(text) => (text.clone(), "text", "text"), + value => (serde_json::to_string_pretty(value)?, "json", "json"), + }; + let mut fields = BTreeMap::from([ + ("source".into(), Value::String(turn.source.clone())), + ("step_id".into(), json!(turn.id)), + (MESSAGE_ENCODING_KEY.into(), Value::String(encoding.into())), + (STORYLINE_METADATA_KEY.into(), Value::Object(turn_metadata)), + ]); + if let Some(edit_key) = edit_key { + fields.insert("call_id".into(), Value::String(edit_key.into())); + } + Ok(AgenticmdBlock { + header: AgenticmdHeader { + type_name: type_name.into(), + length: body.len(), + fields, + }, + body, + }) +} + +fn agenticmd_to_storyline(doc: &AgenticmdDocument) -> Result { let session_id = doc.session_id.clone().unwrap_or_else(|| "unknown".into()); let agent_id = doc.agent_id.clone().unwrap_or_else(|| "unknown".into()); @@ -79,7 +198,7 @@ pub fn agenticmd_to_storyline(doc: &AgenticmdDocument) -> Result Result Result { - story.validate()?; - let mut blocks = Vec::new(); - for turn in &story.turns { - let body = message_text(&turn.message).unwrap_or_default(); - let mut fields = BTreeMap::new(); - fields.insert("source".into(), json!(turn.source)); - fields.insert("step_id".into(), json!(turn.id)); - let kind = turn - .kind - .clone() - .unwrap_or_else(|| turn.effective_kind().to_string()); - fields.insert("kind".into(), json!(kind)); - if let Some(model) = &turn.model_name { - fields.insert("model".into(), json!(model)); - } - if let Some(ms) = turn.latency_ms { - fields.insert("latency_ms".into(), json!(ms)); - } - if let Some(ms) = turn.ttft_ms { - fields.insert("ttft_ms".into(), json!(ms)); - } - if let Some(ts) = &turn.timestamp { - fields.insert("timestamp".into(), json!(ts)); - } - restore_agenticmd_extra_fields(&mut fields, turn.extra.as_ref()); - - let type_name = turn - .extra - .as_ref() - .and_then(|e| e.get("block_type")) - .and_then(|v| v.as_str()) - .unwrap_or("text") - .to_string(); - - blocks.push(AgenticmdBlock { - header: AgenticmdHeader { - type_name, - length: body.len(), - fields, - }, - body, - }); - } +#[cfg(test)] +mod tests { + use super::{encode_agenticmd, parse_agenticmd}; + use crate::{FieldPresence, StoryLink, StorylineDocument, StorylineToolCall, StorylineTurn}; + use serde_json::json; - Ok(AgenticmdDocument { - format: AGENTICMD_FORMAT_NAME.into(), - frontmatter_format: AGENTICMD_FRONTMATTER_FORMAT.into(), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - frontmatter: BTreeMap::new(), - blocks, - }) -} - -fn agenticmd_block_extra(block: &AgenticmdBlock) -> Value { - let mut extra = Map::new(); - extra.insert("block_type".into(), json!(&block.header.type_name)); - for key in EXTRA_CORRELATION_KEYS { - if let Some(v) = block.header.fields.get(*key) { - extra.insert((*key).into(), v.clone()); - } - } - // Prefer header session/agent when present (document-level may be unset). - for key in ["session_id", "agent_id"] { - if let Some(v) = block.header.fields.get(key) { - extra.insert(key.into(), v.clone()); - } - } - Value::Object(extra) -} + #[test] + fn agenticmd_storyline_roundtrip_preserves_the_authoritative_model() { + let mut story = StorylineDocument::new("session-1", "agent-1"); + story.schema_version = Some("ATIF-v1.7".into()); + story.run_id = Some("run-1".into()); + story.attempt_id = Some("attempt-1".into()); + story.agent.version = Some("1.2".into()); + story.agent.model_name = Some("model-1".into()); + story.agent.tool_definitions = Some(json!([{"name":"lookup"}])); + story.agent.extra = Some(json!({"team":"infra"})); + story.parent = Some(StoryLink { + parent_session_id: "parent-1".into(), + spawn_call_id: Some("spawn-1".into()), + spawn_id: Some(9), + relation: "spawn".into(), + }); + story.child_session_ids = Some(vec!["child-1".into()]); + story.notes = Some("readable trajectory".into()); + story.final_metrics = Some(json!({"score": 1})); + story.continued_trajectory_ref = Some("next-1".into()); + story.extra = Some(json!({"unknown":null})); + story.turns.push(StorylineTurn { + id: 7, + kind: Some("autonomous".into()), + timestamp: Some("2026-08-17T01:02:03Z".into()), + source: "agent".into(), + message: json!([{"type":"text","text":"hello"}]), + reasoning_content: Some("reason".into()), + reasoning_effort: Some(json!("high")), + tool_calls: Some(vec![StorylineToolCall { + tool_call_id: "call-1".into(), + function_name: "lookup".into(), + arguments: json!({"q":"x"}), + result: FieldPresence::Null, + duration_ms: Some(12), + extra: Some(json!({"provider":"test"})), + }]), + observation: Some(json!({ + "results":[{"source_call_id":"call-1","content":"ok"}] + })), + metrics: Some(json!({"tokens":3})), + model_name: Some("model-1".into()), + llm_call_count: Some(1), + is_copied_context: Some(false), + latency_ms: Some(50), + ttft_ms: Some(5), + extra: Some(json!({"trace_id":"trace-1"})), + }); -fn restore_agenticmd_extra_fields(fields: &mut BTreeMap, extra: Option<&Value>) { - let Some(extra) = extra.and_then(|v| v.as_object()) else { - return; - }; - for key in EXTRA_CORRELATION_KEYS - .iter() - .chain(["session_id", "agent_id"].iter()) - { - if let Some(v) = extra.get(*key) { - fields.insert((*key).into(), v.clone()); - } + let markdown = encode_agenticmd(&story).unwrap(); + let restored = parse_agenticmd(&markdown).unwrap(); + assert_eq!(restored, story); } } diff --git a/crates/persisting-pchronicle/src/agenticmd/frontmatter.rs b/crates/persisting-pchronicle/src/agenticmd/frontmatter.rs deleted file mode 100644 index 24cda7ac..00000000 --- a/crates/persisting-pchronicle/src/agenticmd/frontmatter.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Typed metadata contract for AgenticMD trajectory frontmatter. - -use serde::{Deserialize, Serialize}; - -use super::codec::{ - encode_agenticmd_preamble, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, -}; -use crate::Result; - -/// Producer/client provenance embedded in an AgenticMD document. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct AgenticmdClientMeta { - pub peer: String, - pub peer_port: u16, - pub pid: u32, - pub command: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub machine_fp: Option, -} - -/// Best-effort session rollup using Storyline-compatible field names. -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -pub struct AgenticmdSessionFrontmatter { - #[serde(rename = "session_id")] - pub session: String, - #[serde(rename = "agent_id")] - pub agent: String, - #[serde(rename = "model_name", skip_serializing_if = "Option::is_none")] - pub model: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub provider: Option, - #[serde(rename = "started_at", skip_serializing_if = "Option::is_none")] - pub started: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub duration: Option, - #[serde(rename = "turn_count", default, skip_serializing_if = "is_zero")] - pub turns: u64, - #[serde(default, skip_serializing_if = "is_zero")] - pub total_tokens: u64, - #[serde(skip_serializing_if = "Option::is_none")] - pub estimated_cost_usd: Option, - #[serde( - rename = "child_session_ids", - default, - skip_serializing_if = "Vec::is_empty" - )] - pub subagents: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub client: Option, -} - -#[derive(Serialize)] -struct FrontmatterDocument<'a> { - format: &'static str, - block: &'static str, - #[serde(flatten)] - summary: &'a AgenticmdSessionFrontmatter, -} - -pub fn encode_agenticmd_session_frontmatter( - summary: &AgenticmdSessionFrontmatter, -) -> Result { - encode_agenticmd_preamble(&FrontmatterDocument { - format: AGENTICMD_FRONTMATTER_FORMAT, - block: AGENTICMD_BLOCK_LAYOUT, - summary, - }) -} - -fn is_zero(value: &u64) -> bool { - *value == 0 -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn session_frontmatter_uses_canonical_agenticmd_contract() { - let encoded = encode_agenticmd_session_frontmatter(&AgenticmdSessionFrontmatter { - session: "s1".into(), - agent: "a1".into(), - turns: 2, - client: Some(AgenticmdClientMeta { - peer: "127.0.0.1:1234".into(), - peer_port: 1234, - pid: 42, - command: "agent".into(), - machine_fp: None, - }), - ..Default::default() - }) - .unwrap(); - assert!(encoded.contains("format: persisting")); - assert!(encoded.contains("session_id: s1")); - assert!(encoded.contains("turn_count: 2")); - assert!(encoded.contains("client:")); - assert!(!encoded.contains("total_tokens:")); - } - - #[test] - fn legacy_short_frontmatter_names_are_rejected() { - let legacy = serde_json::json!({ - "session": "s1", - "agent": "a1" - }); - assert!(serde_json::from_value::(legacy).is_err()); - } -} diff --git a/crates/persisting-pchronicle/src/agenticmd/fs.rs b/crates/persisting-pchronicle/src/agenticmd/fs.rs index a1966456..6bf3eef4 100644 --- a/crates/persisting-pchronicle/src/agenticmd/fs.rs +++ b/crates/persisting-pchronicle/src/agenticmd/fs.rs @@ -13,10 +13,11 @@ use super::codec::{ parse_agenticmd_document, AgenticmdBlock, AgenticmdBlockSpan, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, }; -use super::mapping::agenticmd_block_to_replay_json; +use super::convert::{encode_storyline_preamble, storyline_turn_block}; use super::validate::{ block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name, }; +use crate::{StorylineDocument, StorylineTurn}; /// Diagnostic index of one AgenticMD document. #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -126,6 +127,43 @@ pub fn write_agenticmd_document( write_atomic(path, output.as_bytes()) } +/// Atomically replace an AgenticMD view from its authoritative Storyline model. +pub fn write_agenticmd_storyline(path: &Path, story: &StorylineDocument) -> Result<()> { + let encoded = super::convert::encode_agenticmd(story) + .map_err(|error| anyhow::anyhow!("agenticmd encode: {error}"))?; + write_atomic(path, encoded.as_bytes()) +} + +/// Insert or replace one Storyline turn in a live AgenticMD view. +/// +/// `edit_key` is a syntax-only locator used for streaming draft replacement. It +/// is never copied into the parsed [`StorylineTurn`] or its `extra` field. +pub fn upsert_agenticmd_turn( + path: &Path, + document_meta: &StorylineDocument, + turn: &StorylineTurn, + edit_key: &str, +) -> Result { + if edit_key.trim().is_empty() { + bail!("edit_key must not be empty for AgenticMD upsert"); + } + let mut candidate = document_meta.clone(); + candidate.turns = vec![turn.clone()]; + candidate + .validate() + .map_err(|error| anyhow::anyhow!("invalid Storyline turn: {error}"))?; + + let block = storyline_turn_block(turn, Some(edit_key)) + .map_err(|error| anyhow::anyhow!("agenticmd turn encode: {error}"))?; + if !path.exists() { + let preamble = encode_storyline_preamble(document_meta) + .map_err(|error| anyhow::anyhow!("agenticmd preamble encode: {error}"))?; + write_agenticmd_document(path, &preamble, std::slice::from_ref(&block))?; + return Ok(false); + } + upsert_block_by_call_id(path, edit_key, block) +} + /// Replace only the YAML preamble while preserving every encoded block byte-for-byte. pub fn rewrite_agenticmd_preamble(path: &Path, preamble: &str) -> Result<()> { let content = @@ -137,21 +175,17 @@ pub fn rewrite_agenticmd_preamble(path: &Path, preamble: &str) -> Result<()> { write_atomic(path, &output) } -/// Convert a page of parsed AgenticMD blocks to canonical replay JSON records. -pub fn agenticmd_replay_json_lines( - blocks: &[AgenticmdBlock], - offset: usize, - limit: Option, -) -> Result> { - let end = limit - .map(|limit| offset.saturating_add(limit).min(blocks.len())) - .unwrap_or(blocks.len()); - blocks - .get(offset..end) - .unwrap_or(&[]) - .iter() - .map(agenticmd_block_to_replay_json) - .collect() +/// Replace only the Storyline document metadata in an AgenticMD file. +/// +/// Existing encoded turns remain byte-for-byte intact, including private live +/// edit locators used to replace streaming drafts. +pub fn rewrite_agenticmd_storyline_metadata( + path: &Path, + document_meta: &StorylineDocument, +) -> Result<()> { + let preamble = encode_storyline_preamble(document_meta) + .map_err(|error| anyhow::anyhow!("agenticmd preamble encode: {error}"))?; + rewrite_agenticmd_preamble(path, &preamble) } /// List AgenticMD candidates directly below a run directory. @@ -715,20 +749,54 @@ mod tests { } #[test] - fn structural_scan_and_replay_paging_are_storage_primitives() { + fn structural_scan_reports_excessive_blank_lines() { assert_eq!( agenticmd_structural_issues("a\n\n\n\nb"), vec!["excessive_blank_lines"] ); - let blocks = vec![ - block_with_call("c1", "user", "one"), - block_with_call("c2", "assistant", "two"), - ]; - let replay = agenticmd_replay_json_lines(&blocks, 1, Some(1)).unwrap(); - assert_eq!(replay.len(), 1); - assert!(replay[0].contains("\"call_id\":\"c2\"")); - assert!(agenticmd_replay_json_lines(&blocks, usize::MAX, None) + } + + #[test] + fn storyline_upsert_replaces_draft_without_exposing_edit_key() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("session.md"); + let story = crate::StorylineDocument::new("session-1", "agent-1"); + let mut turn = crate::StorylineTurn { + id: 7, + kind: Some("llm.response.stream".into()), + timestamp: Some("2026-01-01T00:00:00Z".into()), + source: "agent".into(), + message: serde_json::json!("draft"), + reasoning_content: None, + reasoning_effort: None, + tool_calls: None, + observation: None, + metrics: None, + model_name: Some("model-1".into()), + llm_call_count: Some(1), + is_copied_context: None, + latency_ms: None, + ttft_ms: Some(12), + extra: Some(serde_json::json!({"domain": "kept"})), + }; + + assert!(!upsert_agenticmd_turn(&path, &story, &turn, "call-7").unwrap()); + turn.kind = Some("llm.response".into()); + turn.message = serde_json::json!("complete"); + turn.latency_ms = Some(42); + assert!(upsert_agenticmd_turn(&path, &story, &turn, "call-7").unwrap()); + + let parsed = + super::super::convert::parse_agenticmd(&std::fs::read_to_string(&path).unwrap()) + .unwrap(); + assert_eq!(parsed.turns, vec![turn]); + assert_eq!( + parsed.turns[0].extra, + Some(serde_json::json!({"domain": "kept"})) + ); + assert!(index_agenticmd_path(&path) .unwrap() - .is_empty()); + .call_ids + .contains("call-7")); } } diff --git a/crates/persisting-pchronicle/src/agenticmd/mapping/fields.rs b/crates/persisting-pchronicle/src/agenticmd/mapping/fields.rs deleted file mode 100644 index a487eb6c..00000000 --- a/crates/persisting-pchronicle/src/agenticmd/mapping/fields.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::collections::BTreeMap; - -use anyhow::Result; -use serde_json::{json, Value}; - -use crate::formats::events::EventRecord; - -use super::text::{compact_json, content_to_string, visible_assistant_text, visible_user_text}; - -pub(super) fn role_and_body(rec: &EventRecord) -> Result<(String, String)> { - Ok(match rec.kind.as_str() { - "llm.request" | "http.request" => { - ("user".into(), visible_user_text(rec).unwrap_or_default()) - } - "llm.response" | "llm.response.stream" | "http.response" | "http.response.stream" => ( - "assistant".into(), - visible_assistant_text(rec).unwrap_or_default(), - ), - "user" | "assistant" | "system" | "tool" | "note" => ( - rec.kind.clone(), - rec.payload - .get("content") - .and_then(content_to_string) - .unwrap_or_else(|| compact_json(&rec.payload)), - ), - _ => ("note".into(), compact_json(&rec.payload)), - }) -} - -pub(super) fn attach_subagent_link_fields(fields: &mut BTreeMap, rec: &EventRecord) { - if let Some(id) = &rec.subagent_id { - fields.insert("subagent_id".into(), json!(id)); - } - if let Some(id) = &rec.parent_agent_id { - fields.insert("parent_agent_id".into(), json!(id)); - } - for key in [ - "refs_subagent_ids", - "subagent_trajectories", - "subagent_trajectory", - "spawn_hints", - "spawn_links", - "parent_agent_id", - ] { - if let Some(v) = rec.payload.get(key) { - fields.insert(key.into(), v.clone()); - } - } -} - -pub(super) fn attach_llm_fields(fields: &mut BTreeMap, rec: &EventRecord) { - match rec.kind.as_str() { - "llm.request" | "http.request" => { - if let Some(model) = rec.payload.get("model").and_then(|v| v.as_str()) { - fields.insert("model".into(), json!(model)); - } - if let Some(path) = rec.payload.get("path").and_then(|v| v.as_str()) { - fields.insert("path".into(), json!(path)); - } - } - "llm.response" | "llm.response.stream" | "http.response" | "http.response.stream" => { - if let Some(status) = rec.payload.get("status") { - fields.insert("status".into(), status.clone()); - } - if let Some(usage) = rec - .payload - .get("body") - .and_then(|b| b.get("usage")) - .or_else(|| rec.payload.get("usage")) - { - for key in [ - "prompt_tokens", - "completion_tokens", - "total_tokens", - "input_tokens", - "output_tokens", - ] { - if let Some(v) = usage.get(key) { - fields.insert(key.into(), v.clone()); - } - } - if !fields.contains_key("prompt_tokens") { - if let Some(v) = usage.get("input_tokens") { - fields.insert("prompt_tokens".into(), v.clone()); - } - } - if !fields.contains_key("completion_tokens") { - if let Some(v) = usage.get("output_tokens") { - fields.insert("completion_tokens".into(), v.clone()); - } - } - } - if let Some(v) = rec.payload.get("ttft_ms") { - fields.insert("ttft_ms".into(), v.clone()); - } - } - _ => {} - } -} diff --git a/crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs b/crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs deleted file mode 100644 index 37a324e4..00000000 --- a/crates/persisting-pchronicle/src/agenticmd/mapping/mod.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! EventRecord ⇄ AgenticMD debug-view mapping. -//! -//! New blocks use Storyline-like fields. Reverse conversion exists for explicit -//! imports, but AgenticMD is intentionally not a lossless persistence boundary. - -mod fields; -mod text; - -use anyhow::{Context, Result}; -use serde_json::{json, Value}; - -use crate::formats::events::{EventIdentity, EventRecord}; - -use super::body::{append_subagent_refs_footer, strip_subagent_footer_from_body}; -use super::codec::{AgenticmdBlock, AgenticmdHeader}; - -use fields::{attach_llm_fields, attach_subagent_link_fields, role_and_body}; - -/// Build an agenticmd block from an event record (primary write mapping). -/// -/// Uses JSON-oriented visible-text extraction. For live SSE fidelity, stamp -/// `user_content` / `assistant_content` on the payload before calling, or use -/// [`event_record_to_agenticmd_block_with_text`]. -pub fn event_record_to_agenticmd_block(rec: &EventRecord) -> Result { - let (role, body) = role_and_body(rec)?; - event_record_to_agenticmd_block_with_text(rec, &role, &body) -} - -/// Like [`event_record_to_agenticmd_block`] but uses caller-supplied role/body -/// (e.g. capture's SSE-aware `visible_*` extractors). -pub fn event_record_to_agenticmd_block_with_text( - rec: &EventRecord, - role: &str, - body: &str, -) -> Result { - let source = match role { - "user" => "user", - "assistant" | "agent" => "agent", - _ => "system", - }; - let mut fields = std::collections::BTreeMap::from([ - ("kind".into(), json!(rec.kind)), - ("source".into(), json!(source)), - ("producer".into(), json!(rec.source)), - ("event_seq".into(), json!(rec.seq)), - ("step_id".into(), json!(rec.seq / 2 + 1)), - ]); - if rec.payload.get("draft").and_then(|v| v.as_bool()) == Some(true) { - fields.insert("draft".into(), json!(true)); - } - if let Some(sid) = &rec.session_id { - fields.insert("session_id".into(), json!(sid)); - } - if let Some(aid) = &rec.agent_id { - fields.insert("agent_id".into(), json!(aid)); - } - if let Some(ts) = &rec.timestamp { - fields.insert("timestamp".into(), json!(ts)); - } - if let Some(p) = &rec.parent_uuid { - fields.insert("parent_uuid".into(), json!(p)); - } - if let Some(t) = &rec.trace_id { - fields.insert("trace_id".into(), json!(t)); - } - if let Some(c) = &rec.call_id { - fields.insert("call_id".into(), json!(c)); - } - attach_subagent_link_fields(&mut fields, rec); - attach_llm_fields(&mut fields, rec); - - let body = append_subagent_refs_footer(body, &rec.payload); - Ok(AgenticmdBlock { - header: AgenticmdHeader { - type_name: "markdown".into(), - length: body.len(), - fields, - }, - body, - }) -} - -pub fn agenticmd_block_to_replay_json(block: &AgenticmdBlock) -> Result { - let mut o = serde_json::Map::new(); - o.insert("type".into(), json!(&block.header.type_name)); - o.insert("length".into(), json!(block.header.length)); - for (k, v) in &block.header.fields { - o.insert(k.clone(), v.clone()); - } - o.insert("content".into(), json!(&block.body)); - serde_json::to_string(&Value::Object(o)).context("replay JSON") -} - -/// Reconstruct an [`EventRecord`] from an agenticmd block (primary read mapping). -pub fn agenticmd_block_to_event_record(block: &AgenticmdBlock) -> Result { - let content = strip_subagent_footer_from_body(&block.body); - let kind = block.kind().unwrap_or("markdown").to_string(); - let role = block.role().unwrap_or("note"); - let seq = ["event_seq", "seq"] - .iter() - .find_map(|key| block.header.fields.get(*key).and_then(|v| v.as_u64())) - .or_else(|| block.step_id().and_then(|id| u64::try_from(id).ok())) - .unwrap_or(0); - - let payload = match kind.as_str() { - "llm.request" | "http.request" => { - let mut p = json!({ "body": { "messages": [{"role": role, "content": content}] } }); - if let Some(model) = block.header.fields.get("model").and_then(|v| v.as_str()) { - p["model"] = json!(model); - } - if let Some(path) = block.header.fields.get("path").and_then(|v| v.as_str()) { - p["path"] = json!(path); - } - p - } - "llm.response" | "llm.response.stream" | "http.response" | "http.response.stream" => { - let status = block - .header - .fields - .get("status") - .and_then(|v| v.as_u64()) - .unwrap_or(200); - let mut usage = serde_json::Map::new(); - for key in ["prompt_tokens", "completion_tokens", "total_tokens"] { - if let Some(v) = block.header.fields.get(key) { - usage.insert(key.into(), v.clone()); - } - } - let mut body = serde_json::Map::new(); - body.insert( - "choices".into(), - json!([{"message": {"role": "assistant", "content": content}}]), - ); - if !usage.is_empty() { - body.insert("usage".into(), Value::Object(usage)); - } - json!({ "status": status, "body": Value::Object(body) }) - } - _ => json!({ "role": role, "content": content }), - }; - - Ok(EventRecord { - identity: EventIdentity::default(), - seq, - source: block - .header - .fields - .get("producer") - .and_then(|v| v.as_str()) - .unwrap_or("agenticmd-view") - .into(), - kind, - timestamp: block - .header - .fields - .get("timestamp") - .and_then(|v| v.as_str()) - .map(str::to_string), - session_id: block - .header - .fields - .get("session_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - agent_id: block - .header - .fields - .get("agent_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - parent_uuid: block - .header - .fields - .get("parent_uuid") - .and_then(|v| v.as_str()) - .map(str::to_string), - trace_id: block - .header - .fields - .get("trace_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - call_id: block - .header - .fields - .get("call_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - subagent_id: block - .header - .fields - .get("subagent_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - parent_agent_id: block - .header - .fields - .get("parent_agent_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - branch: None, - parent_call_id: None, - payload, - }) -} - -/// Attach source view metadata for explicit imports. -pub fn enrich_event_from_agenticmd_block( - mut rec: EventRecord, - block: &AgenticmdBlock, -) -> EventRecord { - rec.payload["_agenticmd"] = json!({ - "source": block.source(), - "block_fields": block.header.fields, - }); - rec -} - -/// Map AgenticMD blocks to event records for explicit import. -pub fn agenticmd_blocks_to_event_records(blocks: &[AgenticmdBlock]) -> Result> { - blocks - .iter() - .enumerate() - .map(|(i, block)| { - let rec = - agenticmd_block_to_event_record(block).with_context(|| format!("block[{i}]"))?; - Ok(enrich_event_from_agenticmd_block(rec, block)) - }) - .collect() -} - -/// Parse agenticmd markdown (lenient) into enriched event records. -pub fn markdown_document_to_event_records(doc: &str) -> Result> { - let parsed = super::codec::parse_agenticmd_document(doc)?; - agenticmd_blocks_to_event_records(&parsed.blocks) -} diff --git a/crates/persisting-pchronicle/src/agenticmd/mapping/text.rs b/crates/persisting-pchronicle/src/agenticmd/mapping/text.rs deleted file mode 100644 index 90aa8ec9..00000000 --- a/crates/persisting-pchronicle/src/agenticmd/mapping/text.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! JSON-oriented visible text extraction for [`EventRecord`] mapping. -//! -//! Live SSE extraction stays in capture; stamp `user_content` / `assistant_content` -//! when those paths matter. - -use serde_json::Value; - -use crate::formats::events::EventRecord; - -pub(super) fn content_to_string(v: &Value) -> Option { - match v { - Value::String(s) => Some(s.clone()), - Value::Array(parts) => { - let out: Vec<_> = parts - .iter() - .filter_map(|p| p.get("text").and_then(|t| t.as_str())) - .collect(); - if out.is_empty() { - None - } else { - Some(out.join("\n")) - } - } - _ => None, - } -} - -pub(super) fn compact_json(payload: &Value) -> String { - serde_json::to_string(payload).unwrap_or_else(|_| "{}".to_string()) -} - -fn non_empty(s: &str) -> Option { - if s.trim().is_empty() { - None - } else { - Some(s.to_string()) - } -} - -fn llm_inner_body(payload: &Value) -> Option<&Value> { - payload - .get("body") - .filter(|b| b.is_object() || b.is_array() || b.is_string()) -} - -pub(super) fn visible_user_text(rec: &EventRecord) -> Option { - if let Some(s) = rec.payload.get("user_content").and_then(|v| v.as_str()) { - return non_empty(s); - } - let messages = llm_inner_body(&rec.payload) - .and_then(|b| b.get("messages")) - .or_else(|| rec.payload.get("messages"))? - .as_array()?; - for msg in messages.iter().rev() { - if msg.get("role").and_then(|r| r.as_str()) == Some("user") { - if let Some(text) = msg.get("content").and_then(content_to_string) { - return non_empty(&text); - } - } - } - None -} - -pub(super) fn visible_assistant_text(rec: &EventRecord) -> Option { - if let Some(s) = rec - .payload - .get("assistant_content") - .and_then(|v| v.as_str()) - { - return non_empty(s); - } - llm_inner_body(&rec.payload) - .and_then(|b| b.get("choices")) - .or_else(|| rec.payload.get("body").and_then(|b| b.get("choices"))) - .or_else(|| rec.payload.get("choices")) - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .and_then(|c| c.get("message")) - .and_then(|m| m.get("content")) - .and_then(content_to_string) - .or_else(|| { - llm_inner_body(&rec.payload) - .and_then(|b| b.get("content")) - .and_then(content_to_string) - }) - .or_else(|| rec.payload.get("content").and_then(content_to_string)) - .and_then(|s| non_empty(&s)) -} diff --git a/crates/persisting-pchronicle/src/agenticmd/mod.rs b/crates/persisting-pchronicle/src/agenticmd/mod.rs index 5d2f3013..b3464e66 100644 --- a/crates/persisting-pchronicle/src/agenticmd/mod.rs +++ b/crates/persisting-pchronicle/src/agenticmd/mod.rs @@ -1,52 +1,27 @@ //! AgenticMD debug-view domain: codec, mapping, paths, filesystem I/O, and projections. -mod body; mod codec; mod convert; -mod frontmatter; mod fs; mod layout; -mod mapping; #[cfg(feature = "lance-store")] mod projection; mod validate; -pub use body::{ - append_subagent_refs_footer, is_subagent_footer_line, strip_subagent_footer_from_body, -}; -pub use codec::{ - agenticmd_body_byte_offset, encode_agenticmd_block, encode_agenticmd_document, - encode_agenticmd_preamble, parse_agenticmd_blocks_with_spans, parse_agenticmd_document, - AgenticmdBlock, AgenticmdBlockSpan, AgenticmdDocument, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, - AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, -}; -pub use convert::{agenticmd_to_storyline, storyline_to_agenticmd}; -pub use frontmatter::{ - encode_agenticmd_session_frontmatter, AgenticmdClientMeta, AgenticmdSessionFrontmatter, -}; +pub use convert::{encode_agenticmd, parse_agenticmd}; pub use fs::{ - agenticmd_block_count, agenticmd_replay_json_lines, agenticmd_structural_issues, - append_agenticmd_blocks, count_agenticmd_role, encode_agenticmd_block_validated, - find_block_by_call_id_and_role, index_agenticmd_path, list_agenticmd_paths, - parse_agenticmd_document_validated, parse_agenticmd_spans_validated, - read_agenticmd_blocks_from_file, rewrite_agenticmd_preamble, rewrite_block_range, - upsert_block_by_call_id, write_agenticmd_document, AgenticmdFileIndex, + agenticmd_block_count, agenticmd_structural_issues, count_agenticmd_role, index_agenticmd_path, + list_agenticmd_paths, rewrite_agenticmd_storyline_metadata, upsert_agenticmd_turn, + write_agenticmd_storyline, AgenticmdFileIndex, }; pub use layout::{ is_subagent_session_storage_key, is_trajectory_markdown_path, locate_run_bucket_markdown, locate_session_markdown, locate_session_markdown_for_key, sanitize_session_filename, session_markdown_filename, session_markdown_path_for_key, session_markdown_write_path_for_key, }; -pub use mapping::{ - agenticmd_block_to_event_record, agenticmd_block_to_replay_json, - agenticmd_blocks_to_event_records, enrich_event_from_agenticmd_block, - event_record_to_agenticmd_block, event_record_to_agenticmd_block_with_text, - markdown_document_to_event_records, -}; #[cfg(feature = "lance-store")] pub use projection::{ - event_records_to_markdown_blocks, layer_stats, materialize_lance_to_markdown, + event_records_to_storyline, layer_stats, materialize_lance_to_markdown, materialize_markdown_path, write_markdown_projection, LayerStats, MaterializeOutcome, MaterializeStats, }; -pub use validate::{block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name}; diff --git a/crates/persisting-pchronicle/src/agenticmd/projection.rs b/crates/persisting-pchronicle/src/agenticmd/projection.rs index 60a1ebb0..b5c31db5 100644 --- a/crates/persisting-pchronicle/src/agenticmd/projection.rs +++ b/crates/persisting-pchronicle/src/agenticmd/projection.rs @@ -5,14 +5,12 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; -use crate::{EventRecord, RawEventLanceStore, StoryCoords}; - -use super::codec::{ - encode_agenticmd_preamble, AgenticmdBlock, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, +use crate::{ + project_event_records, EventRecord, RawEventLanceStore, StoryCoords, StorylineDocument, }; -use super::fs::{agenticmd_block_count, write_agenticmd_document}; + +use super::fs::{agenticmd_block_count, write_agenticmd_storyline}; use super::layout::{locate_session_markdown_for_key, session_markdown_path_for_key}; -use super::mapping::event_record_to_agenticmd_block; #[derive(Debug, Clone, PartialEq, Eq)] pub struct MaterializeStats { @@ -37,27 +35,36 @@ pub struct LayerStats { pub note: String, } -#[derive(serde::Serialize)] -struct ProjectionPreamble { - format: &'static str, - block: &'static str, -} - pub fn materialize_markdown_path(run_dir: &Path, session_key: &str) -> PathBuf { locate_session_markdown_for_key(run_dir, session_key) .unwrap_or_else(|| session_markdown_path_for_key(run_dir, session_key)) } -pub fn event_records_to_markdown_blocks( +pub fn event_records_to_storyline( records: &[EventRecord], -) -> Result<(Vec, MaterializeStats)> { - let blocks = project_dialogue_blocks(records)?; +) -> Result<(StorylineDocument, MaterializeStats)> { + let eligible = project_dialogue_records(records); + let story = if eligible.is_empty() { + let session_id = records + .iter() + .rev() + .find_map(|record| record.session_id.as_deref()) + .unwrap_or("unknown"); + let agent_id = records + .iter() + .rev() + .find_map(|record| record.agent_id.as_deref()) + .unwrap_or("unknown"); + StorylineDocument::new(session_id, agent_id) + } else { + project_event_records(&eligible)? + }; let stats = MaterializeStats { source_events: records.len(), - markdown_blocks: blocks.len(), - skipped_events: records.len().saturating_sub(blocks.len()), + markdown_blocks: story.turns.len(), + skipped_events: records.len().saturating_sub(eligible.len()), }; - Ok((blocks, stats)) + Ok((story, stats)) } /// Best-effort event → dialogue projection for human inspection. @@ -65,10 +72,10 @@ pub fn event_records_to_markdown_blocks( /// This deliberately drops transport-only and duplicate streaming events. /// AgenticMD is not a persistence boundary, so callers must retain the source /// events or Storyline document when lossless replay is required. -fn project_dialogue_blocks(records: &[EventRecord]) -> Result> { +fn project_dialogue_records(records: &[EventRecord]) -> Vec { let mut last_user_message_count = 0usize; let mut skipped_call_ids = HashSet::new(); - let mut blocks = Vec::new(); + let mut eligible = Vec::new(); for record in records { let skip = match record.kind.as_str() { "llm.request" | "http.request" => { @@ -114,22 +121,15 @@ fn project_dialogue_blocks(records: &[EventRecord]) -> Result false, }; if !skip { - let block = event_record_to_agenticmd_block(record)?; - if !block.body.trim().is_empty() || record.kind == "llm.spawn_link" { - blocks.push(block); - } + eligible.push(record.clone()); } } - Ok(blocks) + eligible } pub fn write_markdown_projection(path: &Path, records: &[EventRecord]) -> Result { - let (blocks, stats) = event_records_to_markdown_blocks(records)?; - let preamble = encode_agenticmd_preamble(&ProjectionPreamble { - format: AGENTICMD_FRONTMATTER_FORMAT, - block: AGENTICMD_BLOCK_LAYOUT, - })?; - write_agenticmd_document(path, &preamble, &blocks) + let (story, stats) = event_records_to_storyline(records)?; + write_agenticmd_storyline(path, &story) .with_context(|| format!("write markdown projection {}", path.display()))?; Ok(stats) } diff --git a/crates/persisting-pchronicle/src/convert/mod.rs b/crates/persisting-pchronicle/src/convert/mod.rs index cf2792fb..eac4d3a9 100644 --- a/crates/persisting-pchronicle/src/convert/mod.rs +++ b/crates/persisting-pchronicle/src/convert/mod.rs @@ -17,7 +17,7 @@ mod atif; mod events; mod openai_msg; -pub use crate::agenticmd::{agenticmd_to_storyline, storyline_to_agenticmd}; +pub use crate::agenticmd::{encode_agenticmd, parse_agenticmd}; pub use actf::{ actf_to_storyline, actf_to_storylines, is_actf_storyline, storyline_to_actf, storylines_to_actf, }; @@ -32,9 +32,7 @@ use crate::format::ChronicleFormat; use crate::formats::actf::ActfDocument; use crate::formats::events::events_lance_only_error; use crate::formats::storyline::StorylineDocument; -use crate::formats::{ - parse_agenticmd_document, parse_openai_msg_document, parse_storyline_document, -}; +use crate::formats::{parse_openai_msg_document, parse_storyline_document}; use crate::Result; /// Parse a supported **string** document into the storyline hub. @@ -53,10 +51,7 @@ pub fn into_storyline(format: ChronicleFormat, input: &str) -> Result Err(events_lance_only_error()), - ChronicleFormat::Agenticmd => { - let doc = parse_agenticmd_document(input)?; - agenticmd_to_storyline(&doc) - } + ChronicleFormat::Agenticmd => parse_agenticmd(input), ChronicleFormat::OpenaiMsg => { let doc = parse_openai_msg_document(input)?; openai_msg_to_storyline(&doc) @@ -74,10 +69,7 @@ pub fn from_storyline(format: ChronicleFormat, story: &StorylineDocument) -> Res ChronicleFormat::Atif => Ok(serde_json::to_string_pretty(&storyline_to_atif(story)?)?), ChronicleFormat::Actf => storyline_to_actf(story)?.to_json_string_pretty(), ChronicleFormat::Events => Err(events_lance_only_error()), - ChronicleFormat::Agenticmd => { - let doc = storyline_to_agenticmd(story)?; - crate::formats::encode_agenticmd_document(&doc) - } + ChronicleFormat::Agenticmd => encode_agenticmd(story), ChronicleFormat::OpenaiMsg => { let doc = storyline_to_openai_msg(story)?; Ok(serde_json::to_string_pretty(&doc)?) diff --git a/crates/persisting-pchronicle/src/formats/mod.rs b/crates/persisting-pchronicle/src/formats/mod.rs index b8f6604c..8377a19c 100644 --- a/crates/persisting-pchronicle/src/formats/mod.rs +++ b/crates/persisting-pchronicle/src/formats/mod.rs @@ -8,21 +8,6 @@ pub mod openai_corpus; pub mod openai_msg; pub mod storyline; -pub use crate::agenticmd::{ - agenticmd_body_byte_offset, encode_agenticmd_block, encode_agenticmd_document, - encode_agenticmd_preamble, parse_agenticmd_blocks_with_spans, parse_agenticmd_document, - AgenticmdBlock, AgenticmdBlockSpan, AgenticmdDocument, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, - AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, -}; -pub use crate::agenticmd::{ - append_subagent_refs_footer, is_subagent_footer_line, strip_subagent_footer_from_body, -}; -pub use crate::agenticmd::{ - block_speaker, validate_agenticmd_block, validate_speaker, validate_type_name, -}; -pub use crate::agenticmd::{ - encode_agenticmd_session_frontmatter, AgenticmdClientMeta, AgenticmdSessionFrontmatter, -}; pub use actf::{ parse_actf_document, ActfAssistantContent, ActfAttempt, ActfDocument, ActfMetric, ActfObservation, ActfStep, ActfToolCall, ActfTrajectory, ACTF_SCHEMA_VERSION, diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index 32728cd7..7747f123 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -43,10 +43,10 @@ pub mod search; pub mod store; pub use agenticmd::{ - agenticmd_block_to_event_record, agenticmd_block_to_replay_json, - agenticmd_blocks_to_event_records, enrich_event_from_agenticmd_block, - event_record_to_agenticmd_block, event_record_to_agenticmd_block_with_text, - markdown_document_to_event_records, + agenticmd_block_count, agenticmd_structural_issues, count_agenticmd_role, encode_agenticmd, + index_agenticmd_path, list_agenticmd_paths, parse_agenticmd, + rewrite_agenticmd_storyline_metadata, upsert_agenticmd_turn, write_agenticmd_storyline, + AgenticmdFileIndex, }; #[cfg(feature = "lance-store")] pub use append_queue::{ @@ -69,21 +69,14 @@ pub use discovery::{ pub use error::{classify_error, Error, ErrorCode, Result}; pub use format::{ChronicleFormat, DocumentFormat}; pub use formats::{ - agenticmd_body_byte_offset, append_subagent_refs_footer, block_speaker, detect_format, - encode_agenticmd_block, encode_agenticmd_document, encode_agenticmd_preamble, - encode_agenticmd_session_frontmatter, events_lance_only_message, export_events_json_pretty, - export_events_jsonl, is_subagent_footer_line, parse_agenticmd_blocks_with_spans, - parse_agenticmd_document, parse_openai_msg_document, parse_storyline_document, - strip_subagent_footer_from_body, validate_agenticmd_block, validate_speaker, - validate_type_name, AgenticmdBlock, AgenticmdBlockSpan, AgenticmdClientMeta, AgenticmdDocument, - AgenticmdHeader, AgenticmdSessionFrontmatter, ChronicleEventRecordExt, EventIdentity, + detect_format, events_lance_only_message, export_events_json_pretty, export_events_jsonl, + parse_openai_msg_document, parse_storyline_document, ChronicleEventRecordExt, EventIdentity, EventRecord, EventsDocument, FieldPresence, LlmCandidate, LlmContentPart, LlmExtensions, LlmGenerationParams, LlmImageSource, LlmMessage, LlmProtocol, LlmRequest, LlmRequestEventPayload, LlmResponse, LlmResponseEventPayload, LlmResponseFormat, LlmRole, LlmStreamEvent, LlmToolChoice, LlmToolChoiceMode, LlmToolDefinition, LlmUsage, OpenaiMsgCorpusReader, OpenaiMsgDocument, OpenaiMsgStep, RecoveredOpenaiMsgFile, StoryLink, - StorylineAgent, StorylineDocument, StorylineToolCall, StorylineTurn, AGENTICMD_BLOCK_LAYOUT, - AGENTICMD_FORMAT_NAME, AGENTICMD_FRONTMATTER_FORMAT, BLOCK_MARKER, + StorylineAgent, StorylineDocument, StorylineToolCall, StorylineTurn, }; pub use formats::{ is_lossless_openai_storyline, parse_openai_msg_corpus_value, recover_openai_msg_files, @@ -111,7 +104,7 @@ pub use operations::bridge::{ pub use operations::dispatch::invoke_request_body; #[cfg(feature = "lance-store")] pub use projection::{ - build_storyline_projection, canonical_projection_lineage, event_records_to_markdown_blocks, + build_storyline_projection, canonical_projection_lineage, event_records_to_storyline, layer_stats, materialize_lance_to_markdown, materialize_markdown_path, projection_lineage_is_fresh, rebuild_storyline_projection, storyline_projection_status, sync_storyline_projection, verify_storyline_projection, write_markdown_projection, LayerStats, @@ -125,16 +118,6 @@ pub use revision::{read_revisions, revision_dataset_path, write_revisions, Revis pub use search::agent as agent_search; #[cfg(feature = "lance-store")] pub use store::maintain_raw_events; -pub use store::{ - agenticmd_block_count, agenticmd_replay_json_lines, agenticmd_structural_issues, - append_agenticmd_blocks, count_agenticmd_role, encode_agenticmd_block_validated, - find_block_by_call_id_and_role, index_agenticmd_path, list_agenticmd_paths, - parse_agenticmd_document_validated, parse_agenticmd_spans_validated, - read_agenticmd_blocks_from_file, rewrite_agenticmd_preamble, rewrite_block_range, - upsert_block_by_call_id, write_agenticmd_document, AgenticmdFileIndex, StoryRunRow, - StoryStepRow, StoryToolCallRow, StorylineTables, STORY_RUNS_TABLE, STORY_STEPS_TABLE, - STORY_TOOL_CALLS_TABLE, -}; #[cfg(feature = "lance-store")] pub use store::{ attempt_registry_now_ms, distinct_session_ids_in_run, event_record_to_event_row, @@ -173,6 +156,10 @@ pub use store::{ #[cfg(feature = "lance-store")] pub use store::{detect_local_query_format, detect_local_query_manifest}; pub use store::{reconstruct_storyline, split_storyline}; +pub use store::{ + StoryRunRow, StoryStepRow, StoryToolCallRow, StorylineTables, STORY_RUNS_TABLE, + STORY_STEPS_TABLE, STORY_TOOL_CALLS_TABLE, +}; #[cfg(feature = "search")] pub const PERSISTING_VECTOR_INDEX_NAME: &str = search::search_lance::PERSISTING_VECTOR_INDEX_NAME; diff --git a/crates/persisting-pchronicle/src/projection/mod.rs b/crates/persisting-pchronicle/src/projection/mod.rs index a4bdeb8c..e91f2ea7 100644 --- a/crates/persisting-pchronicle/src/projection/mod.rs +++ b/crates/persisting-pchronicle/src/projection/mod.rs @@ -3,7 +3,7 @@ mod storyline; pub use crate::agenticmd::{ - event_records_to_markdown_blocks, layer_stats, materialize_lance_to_markdown, + event_records_to_storyline, layer_stats, materialize_lance_to_markdown, materialize_markdown_path, write_markdown_projection, LayerStats, MaterializeOutcome, MaterializeStats, }; diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index c8195648..516bbb59 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -38,14 +38,6 @@ mod storyline; #[path = "storyline/model.rs"] mod storyline_model; -pub use crate::agenticmd::{ - agenticmd_block_count, agenticmd_replay_json_lines, agenticmd_structural_issues, - append_agenticmd_blocks, count_agenticmd_role, encode_agenticmd_block_validated, - find_block_by_call_id_and_role, index_agenticmd_path, list_agenticmd_paths, - parse_agenticmd_document_validated, parse_agenticmd_spans_validated, - read_agenticmd_blocks_from_file, rewrite_agenticmd_preamble, rewrite_block_range, - upsert_block_by_call_id, write_agenticmd_document, AgenticmdFileIndex, -}; #[cfg(feature = "lance-store")] pub use atif_datafusion::{ load_atif_trajectories, AtifDataSource, AtifDataSourceOptions, AtifReader, diff --git a/crates/persisting-pchronicle/src/tests.rs b/crates/persisting-pchronicle/src/tests.rs index 4adf99b8..4d4b8030 100644 --- a/crates/persisting-pchronicle/src/tests.rs +++ b/crates/persisting-pchronicle/src/tests.rs @@ -321,125 +321,6 @@ fn export_events_jsonl_debug_roundtrip_via_test_parser() { assert_eq!(doc.session_id.as_deref(), Some("s1")); } -#[test] -fn parse_agenticmd_document_roundtrip() { - use crate::formats::{ - encode_agenticmd_document, parse_agenticmd_document, AgenticmdBlock, AgenticmdDocument, - AgenticmdHeader, - }; - use serde_json::json; - use std::collections::BTreeMap; - let mut fields = BTreeMap::new(); - fields.insert("role".into(), json!("user")); - fields.insert("kind".into(), json!("dialogue")); - let mut doc = AgenticmdDocument::new(vec![AgenticmdBlock { - header: AgenticmdHeader { - type_name: "text".into(), - length: 0, - fields, - }, - body: "hello".into(), - }]); - doc.session_id = Some("sess-1".into()); - doc.agent_id = Some("agent-a".into()); - let text = encode_agenticmd_document(&doc).unwrap(); - assert!(text.contains("format: persisting")); - assert!(text.contains("session_id: sess-1")); - assert!(text.contains("agent_id: agent-a")); - assert!(text.contains("\n\nhello\n"; - let doc = crate::parse_agenticmd_document(text).unwrap(); - assert_eq!(doc.blocks.len(), 1); - assert_eq!(doc.blocks[0].source(), Some("agent")); - assert_eq!(doc.blocks[0].role(), Some("assistant")); - assert_eq!(doc.blocks[0].step_id(), Some(7)); - assert_eq!(doc.blocks[0].header.type_name, "text"); - assert_eq!(doc.blocks[0].body, "hello"); -} - -#[test] -fn agenticmd_accepts_plain_markdown_but_rejects_unclosed_frontmatter() { - use crate::{parse_agenticmd_blocks_with_spans, parse_agenticmd_document}; - - let unclosed = "---\nformat: persisting\n"; - let err = parse_agenticmd_document(unclosed).unwrap_err(); - assert!( - err.to_string().contains("unclosed YAML frontmatter"), - "{err}" - ); - - let garbage = "---\nformat: persisting\n---\n\nnot a block\n"; - let parsed = parse_agenticmd_document(garbage).unwrap(); - assert_eq!(parsed.blocks.len(), 1); - assert_eq!(parsed.blocks[0].body, "not a block"); - assert_eq!(parsed.blocks[0].source(), Some("system")); - - let spans = parse_agenticmd_blocks_with_spans("---\nformat: persisting\n---\n\n").unwrap(); - assert!(spans.is_empty()); -} - -#[test] -fn agenticmd_body_byte_offset_matches_split() { - use crate::agenticmd_body_byte_offset; - assert_eq!(agenticmd_body_byte_offset("no-fm").unwrap(), 0); - let doc = "---\nformat: persisting\n---\n\nbody"; - let off = agenticmd_body_byte_offset(doc).unwrap(); - assert_eq!(&doc[off..], "\nbody"); - let err = agenticmd_body_byte_offset("---\nno close\n").unwrap_err(); - assert!(err.to_string().contains("unclosed YAML frontmatter")); -} - -#[test] -fn encode_agenticmd_preamble_preserves_nested_mapping() { - use crate::{ - agenticmd_body_byte_offset, encode_agenticmd_preamble, AGENTICMD_BLOCK_LAYOUT, - AGENTICMD_FRONTMATTER_FORMAT, - }; - use serde::Serialize; - - #[derive(Serialize)] - struct Fm<'a> { - format: &'a str, - block: &'a str, - client: Client, - } - #[derive(Serialize)] - struct Client { - peer_port: u16, - command: String, - } - - let preamble = encode_agenticmd_preamble(&Fm { - format: AGENTICMD_FRONTMATTER_FORMAT, - block: AGENTICMD_BLOCK_LAYOUT, - client: Client { - peer_port: 9, - command: "x".into(), - }, - }) - .unwrap(); - assert!(preamble.contains("peer_port: 9")); - let off = agenticmd_body_byte_offset(&preamble).unwrap(); - assert!( - preamble[off..].trim().is_empty(), - "body after preamble should be blank" - ); -} - #[test] fn parse_openai_msg_envelope() { use crate::formats::openai_msg::parse_openai_msg_document; @@ -643,75 +524,6 @@ fn convert_storyline_agenticmd_preserves_dialogue_and_timing() { assert_eq!(turns[1]["ttft_ms"], 7); } -#[test] -fn convert_agenticmd_storyline_preserves_call_id_and_seq() { - use crate::convert::{agenticmd_to_storyline, storyline_to_agenticmd}; - use crate::formats::{AgenticmdBlock, AgenticmdDocument, AgenticmdHeader}; - use serde_json::json; - use std::collections::BTreeMap; - - fn block(role: &str, kind: &str, call_id: &str, seq: u64, body: &str) -> AgenticmdBlock { - let mut fields = BTreeMap::new(); - fields.insert("role".into(), json!(role)); - fields.insert("kind".into(), json!(kind)); - fields.insert("call_id".into(), json!(call_id)); - fields.insert("seq".into(), json!(seq)); - fields.insert("turn".into(), json!(seq / 2 + 1)); - AgenticmdBlock { - header: AgenticmdHeader { - type_name: "markdown".into(), - length: body.len(), - fields, - }, - body: body.into(), - } - } - - let doc = AgenticmdDocument { - format: "agenticmd".into(), - frontmatter_format: "persisting".into(), - session_id: Some("s-cid".into()), - agent_id: Some("a-cid".into()), - frontmatter: BTreeMap::new(), - blocks: vec![ - block("user", "llm.request", "c-42", 0, "hello"), - block("assistant", "llm.response", "c-42", 1, "world"), - ], - }; - let story = agenticmd_to_storyline(&doc).unwrap(); - assert_eq!(story.turns.len(), 2); - assert_eq!( - story.turns[0].extra.as_ref().unwrap()["call_id"], - json!("c-42") - ); - assert_eq!(story.turns[0].extra.as_ref().unwrap()["seq"], json!(0)); - assert_eq!( - story.turns[1].extra.as_ref().unwrap()["call_id"], - json!("c-42") - ); - assert_eq!(story.turns[1].extra.as_ref().unwrap()["seq"], json!(1)); - assert_eq!(story.turns[0].kind.as_deref(), Some("llm.request")); - assert_eq!(story.turns[1].kind.as_deref(), Some("llm.response")); - - let back = storyline_to_agenticmd(&story).unwrap(); - assert_eq!(back.blocks.len(), 2); - assert_eq!( - back.blocks[0].header.fields.get("call_id"), - Some(&json!("c-42")) - ); - assert_eq!(back.blocks[0].header.fields.get("seq"), Some(&json!(0))); - assert_eq!( - back.blocks[1].header.fields.get("call_id"), - Some(&json!("c-42")) - ); - assert_eq!(back.blocks[1].header.fields.get("seq"), Some(&json!(1))); - assert_eq!(back.blocks[0].header.type_name, "markdown"); - assert_eq!( - back.blocks[0].header.fields.get("kind"), - Some(&json!("llm.request")) - ); -} - #[test] fn events_storyline_roundtrip_preserves_call_id_and_seq() { use crate::convert::{events_to_storyline, storyline_to_events}; From ed13df0aa76e3cdf1e0a4fbd414e29f7929ee850 Mon Sep 17 00:00:00 2001 From: Reiase Date: Tue, 18 Aug 2026 00:14:47 +0800 Subject: [PATCH 24/65] feat: unify pchronicle document sources --- crates/persisting-pchronicle/src/document.rs | 81 ++++ crates/persisting-pchronicle/src/lib.rs | 7 + .../src/store/agenticmd_datafusion.rs | 106 +++++ .../src/store/atif_datafusion.rs | 4 + .../src/store/document_source.rs | 435 ++++++++++++++++++ crates/persisting-pchronicle/src/store/mod.rs | 8 + .../tests/document_source.rs | 175 +++++++ 7 files changed, 816 insertions(+) create mode 100644 crates/persisting-pchronicle/src/document.rs create mode 100644 crates/persisting-pchronicle/src/store/agenticmd_datafusion.rs create mode 100644 crates/persisting-pchronicle/src/store/document_source.rs create mode 100644 crates/persisting-pchronicle/tests/document_source.rs diff --git a/crates/persisting-pchronicle/src/document.rs b/crates/persisting-pchronicle/src/document.rs new file mode 100644 index 00000000..1e6507d5 --- /dev/null +++ b/crates/persisting-pchronicle/src/document.rs @@ -0,0 +1,81 @@ +//! Unified read/query entrypoint for pChronicle's physical document formats. + +use std::path::Path; + +use datafusion::prelude::SessionContext; + +use crate::{DocumentFormat, Result, StorylineDocument}; + +/// Static filter pushdown guarantee exposed by a document source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FilterPushdown { + Unsupported, + Inexact, + Exact, + /// The guarantee depends on the concrete expression and table columns. + ExpressionDependent, +} + +/// Logical tables registered by a source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QueryTables { + Events, + Storyline, +} + +/// Truthful optimization capabilities for one opened provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct QueryCapabilities { + pub projection_pushdown: bool, + pub filter_pushdown: FilterPushdown, + pub limit_pushdown: bool, + pub scalar_indexes: bool, + pub streaming_decode: bool, + pub late_content_materialization: bool, + pub snapshot_consistent: bool, +} + +/// Maximum Storyline rows retained by the convenience materialization API. +pub const DEFAULT_DOCUMENT_MATERIALIZE_ROWS: usize = 10_000; +/// Maximum serialized Storyline bytes retained by the convenience materialization API. +pub const DEFAULT_DOCUMENT_MATERIALIZE_BYTES: usize = 64 * 1024 * 1024; + +/// One opened physical document source. Provider variants remain private. +#[derive(Debug)] +pub struct DocumentSource { + pub(crate) inner: crate::store::DocumentSourceImpl, +} + +/// Open one of the six physical pChronicle document formats. +pub async fn open_document(format: DocumentFormat, path: &Path) -> Result { + Ok(DocumentSource { + inner: crate::store::open_document_source(format, path).await?, + }) +} + +impl DocumentSource { + pub fn format(&self) -> DocumentFormat { + self.inner.format() + } + + pub fn capabilities(&self) -> QueryCapabilities { + self.inner.capabilities() + } + + /// Materialize all Storylines, failing closed when the aggregate budget is exceeded. + pub async fn project_storylines(&self) -> Result> { + self.inner.project_storylines().await + } + + /// Visit Storylines one at a time without retaining the complete source. + pub async fn for_each_storyline(&self, on_storyline: F) -> Result<()> + where + F: FnMut(StorylineDocument) -> Result<()>, + { + self.inner.for_each_storyline(on_storyline).await + } + + pub fn register_datafusion(&self, context: &SessionContext) -> Result { + self.inner.register_datafusion(context) + } +} diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index 7747f123..dda7d40c 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -26,6 +26,8 @@ pub mod atif; pub mod convert; #[cfg(feature = "lance-store")] pub mod discovery; +#[cfg(feature = "lance-store")] +pub mod document; pub mod error; pub mod format; pub mod formats; @@ -66,6 +68,11 @@ pub use convert::{ pub use discovery::{ drop_lifecycle_run_partitions, expand_story_locations, expand_story_locations_blocking, }; +#[cfg(feature = "lance-store")] +pub use document::{ + open_document, DocumentSource, FilterPushdown, QueryCapabilities, QueryTables, + DEFAULT_DOCUMENT_MATERIALIZE_BYTES, DEFAULT_DOCUMENT_MATERIALIZE_ROWS, +}; pub use error::{classify_error, Error, ErrorCode, Result}; pub use format::{ChronicleFormat, DocumentFormat}; pub use formats::{ diff --git a/crates/persisting-pchronicle/src/store/agenticmd_datafusion.rs b/crates/persisting-pchronicle/src/store/agenticmd_datafusion.rs new file mode 100644 index 00000000..bc8015b9 --- /dev/null +++ b/crates/persisting-pchronicle/src/store/agenticmd_datafusion.rs @@ -0,0 +1,106 @@ +//! Storyline-schema DataFusion provider for one AgenticMD document. + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use datafusion::catalog::Session; +use datafusion::datasource::{MemTable, TableProvider}; +use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType}; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::SessionContext; +use lance::deps::arrow_schema::SchemaRef; + +use crate::{split_storyline, StorylineDocument}; + +use super::{ + story_runs_arrow_schema, story_runs_to_batch, story_steps_arrow_schema, story_steps_to_batch, + story_tool_calls_arrow_schema, story_tool_calls_to_batch, StorylineDataFusionTableNames, +}; + +#[derive(Debug)] +pub(crate) struct AgenticMdDataSource { + runs: Arc, + steps: Arc, + tool_calls: Arc, +} + +impl AgenticMdDataSource { + pub(crate) fn new(story: &StorylineDocument) -> Result { + let tables = split_storyline(story)?; + Ok(Self { + runs: unsupported_filter_table( + story_runs_arrow_schema(), + story_runs_to_batch(std::slice::from_ref(&tables.run))?, + )?, + steps: unsupported_filter_table( + story_steps_arrow_schema(), + story_steps_to_batch(&tables.steps)?, + )?, + tool_calls: unsupported_filter_table( + story_tool_calls_arrow_schema(), + story_tool_calls_to_batch(&tables.tool_calls)?, + )?, + }) + } + + pub(crate) fn register(&self, context: &SessionContext) -> Result<()> { + let names = StorylineDataFusionTableNames::default(); + context + .register_table(&names.runs, self.runs.clone()) + .context("register AgenticMD runs table")?; + context + .register_table(&names.steps, self.steps.clone()) + .context("register AgenticMD steps table")?; + context + .register_table(&names.tool_calls, self.tool_calls.clone()) + .context("register AgenticMD tool_calls table")?; + Ok(()) + } +} + +fn unsupported_filter_table( + schema: SchemaRef, + batch: lance::deps::arrow_array::RecordBatch, +) -> Result> { + let table = MemTable::try_new(schema, vec![vec![batch]])?; + Ok(Arc::new(AgenticMdTableProvider { + inner: Arc::new(table), + })) +} + +#[derive(Debug)] +struct AgenticMdTableProvider { + inner: Arc, +} + +#[async_trait] +impl TableProvider for AgenticMdTableProvider { + fn schema(&self) -> SchemaRef { + self.inner.schema() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> datafusion::common::Result> { + self.inner.scan(state, projection, &[], None).await + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> datafusion::common::Result> { + Ok(vec![ + TableProviderFilterPushDown::Unsupported; + filters.len() + ]) + } +} diff --git a/crates/persisting-pchronicle/src/store/atif_datafusion.rs b/crates/persisting-pchronicle/src/store/atif_datafusion.rs index 8cf5b654..db3edd08 100644 --- a/crates/persisting-pchronicle/src/store/atif_datafusion.rs +++ b/crates/persisting-pchronicle/src/store/atif_datafusion.rs @@ -249,6 +249,10 @@ impl AtifReader { Ok(Self::from_files(manifest.files())) } + pub(crate) fn from_manifest(manifest: &LocalQueryManifest) -> Self { + Self::from_files(manifest.files()) + } + fn from_files(files: &[LocalQueryInputFile]) -> Self { Self { files: files diff --git a/crates/persisting-pchronicle/src/store/document_source.rs b/crates/persisting-pchronicle/src/store/document_source.rs new file mode 100644 index 00000000..f0b87a1f --- /dev/null +++ b/crates/persisting-pchronicle/src/store/document_source.rs @@ -0,0 +1,435 @@ +//! Private provider variants behind the public `DocumentSource` API. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use datafusion::arrow::array::{Array, StringArray}; +use datafusion::prelude::SessionContext; + +use crate::document::{ + FilterPushdown, QueryCapabilities, QueryTables, DEFAULT_DOCUMENT_MATERIALIZE_BYTES, + DEFAULT_DOCUMENT_MATERIALIZE_ROWS, +}; +use crate::{ + actf_to_storylines, parse_agenticmd, parse_openai_msg_corpus_value, project_event_records, + ChronicleFormat, DocumentFormat, Error, Result, StorylineDocument, +}; + +use super::{ + AgenticMdDataSource, AtifReader, FileTrajectoryDataSource, FileTrajectoryFormat, + LocalQueryManifest, RawEventDataSource, StorylineDataSource, StorylineLanceStore, + DEFAULT_MAX_EVENT_FALLBACK_BYTES, DEFAULT_MAX_EVENT_FALLBACK_ROWS, +}; + +pub(crate) trait QueryDocumentSource { + fn format(&self) -> DocumentFormat; + fn tables(&self) -> QueryTables; + fn capabilities(&self) -> QueryCapabilities; + fn register(&self, context: &SessionContext) -> Result<()>; +} + +#[derive(Debug)] +pub(crate) enum DocumentSourceImpl { + Events { + path: PathBuf, + source: RawEventDataSource, + }, + Storyline { + path: PathBuf, + source: StorylineDataSource, + store: StorylineLanceStore, + }, + AgenticMd { + path: PathBuf, + story: StorylineDocument, + source: AgenticMdDataSource, + }, + Files { + format: DocumentFormat, + path: PathBuf, + manifest: LocalQueryManifest, + source: FileTrajectoryDataSource, + }, +} + +pub(crate) async fn open_document_source( + format: DocumentFormat, + path: &Path, +) -> Result { + let path = path.to_path_buf(); + match format { + DocumentFormat::CanonicalEvent => Ok(DocumentSourceImpl::Events { + source: RawEventDataSource::open(&path).await.map_err(other)?, + path, + }), + DocumentFormat::Storyline => { + let source = StorylineDataSource::open(&path).await.map_err(other)?; + let root = path + .to_str() + .ok_or_else(|| Error::Other("Storyline path is not valid UTF-8".into()))?; + let store = StorylineLanceStore::open_uri_unchecked(root) + .await + .map_err(other)?; + Ok(DocumentSourceImpl::Storyline { + path, + source, + store, + }) + } + DocumentFormat::AgenticMd => { + let input = std::fs::read_to_string(&path).map_err(Error::from)?; + let story = parse_agenticmd(&input).map_err(|error| with_path(error, &path))?; + let source = AgenticMdDataSource::new(&story).map_err(other)?; + Ok(DocumentSourceImpl::AgenticMd { + path, + story, + source, + }) + } + DocumentFormat::Atif | DocumentFormat::OpenaiMsg | DocumentFormat::Actf => { + let legacy_format = match format { + DocumentFormat::Atif => ChronicleFormat::Atif, + DocumentFormat::OpenaiMsg => ChronicleFormat::OpenaiMsg, + DocumentFormat::Actf => ChronicleFormat::Actf, + _ => unreachable!(), + }; + let provider_format = match format { + DocumentFormat::Atif => FileTrajectoryFormat::Atif, + DocumentFormat::OpenaiMsg => FileTrajectoryFormat::OpenaiMsg, + DocumentFormat::Actf => FileTrajectoryFormat::Actf, + _ => unreachable!(), + }; + let manifest = LocalQueryManifest::for_format(&path, legacy_format).map_err(other)?; + let source = + FileTrajectoryDataSource::from_manifest(manifest.clone()).map_err(other)?; + debug_assert_eq!(source.format(), provider_format); + Ok(DocumentSourceImpl::Files { + format, + path, + manifest, + source, + }) + } + } +} + +impl DocumentSourceImpl { + pub(crate) fn format(&self) -> DocumentFormat { + QueryDocumentSource::format(self) + } + + pub(crate) fn capabilities(&self) -> QueryCapabilities { + QueryDocumentSource::capabilities(self) + } + + pub(crate) fn register_datafusion(&self, context: &SessionContext) -> Result { + QueryDocumentSource::register(self, context)?; + Ok(QueryDocumentSource::tables(self)) + } + + pub(crate) async fn project_storylines(&self) -> Result> { + let mut stories = Vec::new(); + let mut retained_rows = 0usize; + let mut retained_bytes = 0usize; + self.for_each_storyline(|story| { + retained_rows = retained_rows + .checked_add(story_rows(&story)) + .ok_or_else(|| budget_error(self, "row count overflow"))?; + if retained_rows > DEFAULT_DOCUMENT_MATERIALIZE_ROWS { + return Err(budget_error( + self, + &format!( + "materialized rows {retained_rows} exceed {DEFAULT_DOCUMENT_MATERIALIZE_ROWS}" + ), + )); + } + retained_bytes = retained_bytes + .checked_add(serde_json::to_vec(&story)?.len()) + .ok_or_else(|| budget_error(self, "byte count overflow"))?; + if retained_bytes > DEFAULT_DOCUMENT_MATERIALIZE_BYTES { + return Err(budget_error( + self, + &format!( + "materialized bytes {retained_bytes} exceed {DEFAULT_DOCUMENT_MATERIALIZE_BYTES}" + ), + )); + } + stories.push(story); + Ok(()) + }) + .await?; + Ok(stories) + } + + pub(crate) async fn for_each_storyline(&self, mut on_storyline: F) -> Result<()> + where + F: FnMut(StorylineDocument) -> Result<()>, + { + match self { + Self::AgenticMd { story, .. } => on_storyline(story.clone()), + Self::Files { + format, manifest, .. + } => for_each_file_storyline(*format, manifest, on_storyline), + Self::Storyline { source, store, .. } => { + let context = SessionContext::new(); + source.register(&context).map_err(other)?; + for session_id in distinct_strings( + &context, + "SELECT session_id FROM runs ORDER BY session_id", + "session_id", + ) + .await? + { + let story = store + .get_storyline_full(&session_id) + .await + .map_err(other)? + .ok_or_else(|| Error::SessionNotFound(session_id.clone()))?; + on_storyline(story)?; + } + Ok(()) + } + Self::Events { source, .. } => { + let context = SessionContext::new(); + source.register(&context).map_err(other)?; + for session_id in distinct_strings( + &context, + "SELECT DISTINCT session_id FROM events WHERE session_id IS NOT NULL ORDER BY session_id", + "session_id", + ) + .await? + { + let requested = BTreeSet::from([session_id]); + let records = source + .read_records_for_storylines_bounded( + &requested, + DEFAULT_MAX_EVENT_FALLBACK_ROWS, + DEFAULT_MAX_EVENT_FALLBACK_BYTES, + ) + .await + .map_err(|error| { + if error.to_string().contains("exceeds max_event_fallback") { + budget_error(self, &error.to_string()) + } else { + other(error) + } + })?; + on_storyline(project_event_records(&records)?)?; + } + Ok(()) + } + } + } + + pub(crate) fn source_count(&self) -> usize { + match self { + Self::Files { source, .. } => source.file_count(), + _ => 1, + } + } + + pub(crate) fn file_metrics(&self) -> Option { + match self { + Self::Files { source, .. } => Some(source.metrics()), + _ => None, + } + } +} + +impl QueryDocumentSource for DocumentSourceImpl { + fn format(&self) -> DocumentFormat { + match self { + Self::Events { .. } => DocumentFormat::CanonicalEvent, + Self::Storyline { .. } => DocumentFormat::Storyline, + Self::AgenticMd { .. } => DocumentFormat::AgenticMd, + Self::Files { format, .. } => *format, + } + } + + fn tables(&self) -> QueryTables { + match self { + Self::Events { .. } => QueryTables::Events, + _ => QueryTables::Storyline, + } + } + + fn capabilities(&self) -> QueryCapabilities { + match self.format() { + DocumentFormat::CanonicalEvent => QueryCapabilities { + projection_pushdown: true, + filter_pushdown: FilterPushdown::Exact, + limit_pushdown: true, + scalar_indexes: true, + streaming_decode: true, + late_content_materialization: false, + snapshot_consistent: true, + }, + DocumentFormat::Storyline => QueryCapabilities { + projection_pushdown: true, + filter_pushdown: FilterPushdown::ExpressionDependent, + limit_pushdown: true, + scalar_indexes: true, + streaming_decode: false, + late_content_materialization: true, + snapshot_consistent: true, + }, + DocumentFormat::Atif => QueryCapabilities { + projection_pushdown: true, + filter_pushdown: FilterPushdown::Inexact, + limit_pushdown: true, + scalar_indexes: false, + streaming_decode: true, + late_content_materialization: false, + snapshot_consistent: false, + }, + DocumentFormat::OpenaiMsg | DocumentFormat::Actf => QueryCapabilities { + projection_pushdown: true, + filter_pushdown: FilterPushdown::Unsupported, + limit_pushdown: true, + scalar_indexes: false, + streaming_decode: false, + late_content_materialization: false, + snapshot_consistent: false, + }, + DocumentFormat::AgenticMd => QueryCapabilities { + projection_pushdown: true, + filter_pushdown: FilterPushdown::Unsupported, + limit_pushdown: false, + scalar_indexes: false, + streaming_decode: false, + late_content_materialization: false, + snapshot_consistent: false, + }, + } + } + + fn register(&self, context: &SessionContext) -> Result<()> { + match self { + Self::Events { source, .. } => source.register(context).map_err(other), + Self::Storyline { source, .. } => source.register(context).map_err(other), + Self::AgenticMd { source, .. } => source.register(context).map_err(other), + Self::Files { source, .. } => source.register(context).map_err(other), + } + } +} + +fn for_each_file_storyline( + format: DocumentFormat, + manifest: &LocalQueryManifest, + mut on_storyline: F, +) -> Result<()> +where + F: FnMut(StorylineDocument) -> Result<()>, +{ + match format { + DocumentFormat::Atif => { + for trajectory in AtifReader::from_manifest(manifest) { + on_storyline(crate::convert::atif_to_storyline( + &trajectory.map_err(other)?, + )?)?; + } + } + DocumentFormat::OpenaiMsg => { + for file in manifest.files() { + file.validate_unchanged().map_err(other)?; + let document = serde_json::from_slice(&std::fs::read(file.path())?)?; + for story in parse_openai_msg_corpus_value(&document, file.relative_path())? { + on_storyline(story)?; + } + file.validate_unchanged().map_err(other)?; + } + } + DocumentFormat::Actf => { + for file in manifest.files() { + file.validate_unchanged().map_err(other)?; + let document = + crate::ActfDocument::from_json_str(&std::fs::read_to_string(file.path())?)?; + for story in actf_to_storylines(&document)? { + on_storyline(story)?; + } + file.validate_unchanged().map_err(other)?; + } + } + _ => unreachable!(), + } + Ok(()) +} + +async fn distinct_strings( + context: &SessionContext, + sql: &str, + column: &str, +) -> Result> { + let batches = context + .sql(sql) + .await + .map_err(other)? + .collect() + .await + .map_err(other)?; + let mut values = Vec::new(); + for batch in batches { + let index = batch.schema().index_of(column).map_err(other)?; + let array = batch + .column(index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::Other(format!("query column '{column}' is not Utf8")))?; + for row in 0..array.len() { + if !array.is_null(row) { + values.push(array.value(row).to_string()); + } + } + } + Ok(values) +} + +fn story_rows(story: &StorylineDocument) -> usize { + 1usize.saturating_add(story.turns.len()).saturating_add( + story + .turns + .iter() + .map(|turn| turn.tool_calls.as_ref().map_or(0, Vec::len)) + .sum::(), + ) +} + +fn budget_error(source: &DocumentSourceImpl, budget: &str) -> Error { + Error::SourceBudgetExceeded { + format: source.format(), + path: Some(source.path().to_path_buf()), + budget: budget.into(), + } +} + +impl DocumentSourceImpl { + fn path(&self) -> &Path { + match self { + Self::Events { path, .. } + | Self::Storyline { path, .. } + | Self::AgenticMd { path, .. } + | Self::Files { path, .. } => path, + } + } +} + +fn other(error: impl std::fmt::Display) -> Error { + Error::Other(error.to_string()) +} + +fn with_path(error: Error, path: &Path) -> Error { + match error { + Error::InvalidDocument { + format, + location, + message, + .. + } => Error::InvalidDocument { + format, + path: Some(path.to_path_buf()), + location, + message, + }, + error => error, + } +} diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 516bbb59..005f7288 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -7,6 +7,8 @@ #[cfg(feature = "lance-store")] use anyhow::Context as _; +#[cfg(feature = "lance-store")] +mod agenticmd_datafusion; #[cfg(feature = "lance-store")] mod atif_datafusion; #[cfg(feature = "lance-store")] @@ -16,6 +18,8 @@ mod catalog; #[cfg(feature = "lance-store")] pub(crate) mod dataset_write_lock; #[cfg(feature = "lance-store")] +mod document_source; +#[cfg(feature = "lance-store")] mod egress; #[cfg(feature = "lance-store")] mod event_row; @@ -38,6 +42,8 @@ mod storyline; #[path = "storyline/model.rs"] mod storyline_model; +#[cfg(feature = "lance-store")] +pub(crate) use agenticmd_datafusion::AgenticMdDataSource; #[cfg(feature = "lance-store")] pub use atif_datafusion::{ load_atif_trajectories, AtifDataSource, AtifDataSourceOptions, AtifReader, @@ -56,6 +62,8 @@ pub use catalog::{ DEFAULT_MAX_EVENT_FALLBACK_ROWS, }; #[cfg(feature = "lance-store")] +pub(crate) use document_source::{open_document_source, DocumentSourceImpl}; +#[cfg(feature = "lance-store")] pub use egress::{export_source_dirs, export_story_bundle, ExportOutcome}; #[cfg(feature = "lance-store")] pub use event_row::{event_record_to_event_row, event_row_to_event_record, EventRow}; diff --git a/crates/persisting-pchronicle/tests/document_source.rs b/crates/persisting-pchronicle/tests/document_source.rs new file mode 100644 index 00000000..a51c4d6d --- /dev/null +++ b/crates/persisting-pchronicle/tests/document_source.rs @@ -0,0 +1,175 @@ +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use datafusion::prelude::SessionContext; +use persisting_pchronicle::{ + encode_agenticmd, open_document, DocumentFormat, Error, EventIdentity, EventRecord, + FilterPushdown, QueryTables, RawEventLanceStore, StoryCoords, StorylineDocument, + StorylineLanceStore, StorylineTurn, DEFAULT_DOCUMENT_MATERIALIZE_ROWS, +}; +use serde_json::json; + +fn fixture(relative: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join(relative) +} + +fn turn(id: i64, message: &str) -> StorylineTurn { + StorylineTurn { + id, + kind: Some("dialogue".into()), + timestamp: None, + source: "user".into(), + message: json!(message), + reasoning_content: None, + reasoning_effort: None, + tool_calls: None, + observation: None, + metrics: None, + model_name: None, + llm_call_count: None, + is_copied_context: None, + latency_ms: None, + ttft_ms: None, + extra: None, + } +} + +async fn assert_storyline_tables(format: DocumentFormat, path: &Path) -> Result<()> { + let source = open_document(format, path).await?; + assert_eq!(source.format(), format); + assert_eq!( + source.register_datafusion(&SessionContext::new())?, + QueryTables::Storyline + ); + assert!(!source.project_storylines().await?.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn opens_all_six_formats_and_reports_true_capabilities() -> Result<()> { + let temporary = tempfile::tempdir()?; + + let agentic_path = temporary.path().join("story.md"); + let mut agentic_story = StorylineDocument::new("agentic-session", "agent"); + agentic_story.turns.push(turn(1, "hello")); + std::fs::write(&agentic_path, encode_agenticmd(&agentic_story)?)?; + + let storyline_path = temporary.path().join("storyline"); + let storyline_store = StorylineLanceStore::open(&storyline_path).await?; + storyline_store + .replace_storylines(std::slice::from_ref(&agentic_story)) + .await?; + + let event_storage = temporary.path().join("events"); + std::fs::create_dir_all(&event_storage)?; + let event_coords = StoryCoords::new( + event_storage.to_string_lossy(), + "agent", + "event-session", + None, + ); + RawEventLanceStore + .append_events( + &event_coords, + &[EventRecord { + identity: EventIdentity::default(), + seq: 1, + source: "test".into(), + kind: "note".into(), + timestamp: Some("2026-01-01T00:00:00Z".into()), + session_id: Some("event-session".into()), + agent_id: Some("agent".into()), + parent_uuid: None, + trace_id: None, + call_id: None, + subagent_id: None, + parent_agent_id: None, + branch: None, + parent_call_id: None, + payload: json!({"content": "event"}), + }], + ) + .await?; + let event_path = persisting_pchronicle::raw_event_lance_path(&event_coords)?; + + let events = open_document(DocumentFormat::CanonicalEvent, &event_path).await?; + assert_eq!( + events.register_datafusion(&SessionContext::new())?, + QueryTables::Events + ); + let event_caps = events.capabilities(); + assert_eq!(event_caps.filter_pushdown, FilterPushdown::Exact); + assert!(event_caps.scalar_indexes); + assert!(event_caps.snapshot_consistent); + assert_eq!(events.project_storylines().await?.len(), 1); + + let storyline = open_document(DocumentFormat::Storyline, &storyline_path).await?; + assert_eq!( + storyline.register_datafusion(&SessionContext::new())?, + QueryTables::Storyline + ); + let storyline_caps = storyline.capabilities(); + assert_eq!( + storyline_caps.filter_pushdown, + FilterPushdown::ExpressionDependent + ); + assert!(storyline_caps.late_content_materialization); + assert_eq!( + storyline.project_storylines().await?, + vec![agentic_story.clone()] + ); + + let agentic = open_document(DocumentFormat::AgenticMd, &agentic_path).await?; + assert_eq!( + agentic.capabilities().filter_pushdown, + FilterPushdown::Unsupported + ); + assert_eq!(agentic.project_storylines().await?, vec![agentic_story]); + + assert_storyline_tables( + DocumentFormat::Atif, + &fixture("tests/fixtures/atif/dialogue_10.json"), + ) + .await?; + assert_storyline_tables( + DocumentFormat::OpenaiMsg, + &fixture("tests/fixtures/import_roundtrip/cybergym_0729001_trimmed.json"), + ) + .await?; + assert_storyline_tables( + DocumentFormat::Actf, + &fixture("tests/fixtures/import_roundtrip/make-doom-for-mips_trimmed.actf.json"), + ) + .await?; + Ok(()) +} + +#[tokio::test] +async fn materialization_budget_fails_closed_but_callback_visits_the_complete_story() -> Result<()> +{ + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("large.md"); + let mut story = StorylineDocument::new("large", "agent"); + story.turns = (0..=DEFAULT_DOCUMENT_MATERIALIZE_ROWS) + .map(|index| turn(index as i64 + 1, "x")) + .collect(); + std::fs::write(&path, encode_agenticmd(&story)?)?; + + let source = open_document(DocumentFormat::AgenticMd, &path).await?; + assert!(matches!( + source.project_storylines().await, + Err(Error::SourceBudgetExceeded { .. }) + )); + let mut visited = Vec::new(); + source + .for_each_storyline(|story| { + visited.push((story.session_id, story.turns.len())); + Ok(()) + }) + .await?; + assert_eq!( + visited, + vec![("large".to_string(), DEFAULT_DOCUMENT_MATERIALIZE_ROWS + 1)] + ); + Ok(()) +} From f48f3a4f729594f37885d16a083d11e483dfc50c Mon Sep 17 00:00:00 2001 From: Reiase Date: Tue, 18 Aug 2026 00:31:37 +0800 Subject: [PATCH 25/65] refactor: converge pchronicle query engine entrypoints --- .../persisting-pchronicle-cli/src/exchange.rs | 4 +- crates/persisting-pchronicle-cli/src/lib.rs | 11 +- .../src/server/acceleration.rs | 2 +- .../src/server/mod.rs | 2 +- crates/persisting-pchronicle-cli/src/tests.rs | 2 +- .../examples/langfuse_backend_feasibility.rs | 51 ++-- crates/persisting-pchronicle/src/lib.rs | 21 +- .../src/store/catalog/mod.rs | 8 + .../src/store/catalog/tests.rs | 27 +- .../src/store/document_source.rs | 98 +++++-- crates/persisting-pchronicle/src/store/mod.rs | 4 +- .../src/store/query_engine.rs | 275 ++++-------------- .../tests/direct_file_query.rs | 217 +++++++++++--- .../tests/production_scale.rs | 22 +- .../tests/query_engine.rs | 207 +++++++++---- .../persisting-pchronicle/tests/s3_storage.rs | 32 +- 16 files changed, 581 insertions(+), 402 deletions(-) diff --git a/crates/persisting-pchronicle-cli/src/exchange.rs b/crates/persisting-pchronicle-cli/src/exchange.rs index a54f6a1b..c7dd973b 100644 --- a/crates/persisting-pchronicle-cli/src/exchange.rs +++ b/crates/persisting-pchronicle-cli/src/exchange.rs @@ -205,7 +205,9 @@ async fn export_from_snapshot( ); let sql = export_address_sql(args)?; - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()) + let engine = snapshot + .clone() + .query_engine(Default::default()) .await .map_err(|error| redact_query_error(&error, &[dataset_uri.to_string()], None))?; let row_limit = args diff --git a/crates/persisting-pchronicle-cli/src/lib.rs b/crates/persisting-pchronicle-cli/src/lib.rs index a478f565..9de38168 100644 --- a/crates/persisting-pchronicle-cli/src/lib.rs +++ b/crates/persisting-pchronicle-cli/src/lib.rs @@ -1359,7 +1359,7 @@ async fn run_status( .unwrap_or_else(|| "Source discovery failed".into()), }) .collect::>(); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; let timeout = Duration::from_secs(args.timeout_seconds); let deadline = tokio::time::Instant::now() + timeout; let counts = match query_status_counts(&engine, None, deadline, timeout).await { @@ -1465,7 +1465,8 @@ async fn run_query( .await?; let snapshot = Arc::new(snapshot); let snapshot_id = snapshot.snapshot_id().to_string(); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot) + let engine = snapshot + .query_engine(Default::default()) .await .map_err(|error| redact_query_error(&error, &dataset_uris, None))?; let mut buffer = LimitedBuffer::new(args.max_output_bytes); @@ -1550,7 +1551,8 @@ async fn run_analysis( .await?; let snapshot = Arc::new(snapshot); let snapshot_id = snapshot.snapshot_id().to_string(); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot) + let engine = snapshot + .query_engine(Default::default()) .await .map_err(|error| redact_query_error(&error, &dataset_uris, None))?; let bounded_sql = format!("{sql}\nLIMIT {}", options.limit); @@ -1731,7 +1733,8 @@ async fn run_find( .context("find Dataset URI missing after discovery")?; let snapshot = Arc::new(snapshot); let snapshot_id = snapshot.snapshot_id().to_string(); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot) + let engine = snapshot + .query_engine(Default::default()) .await .map_err(|error| redact_query_error(&error, std::slice::from_ref(&dataset_uri), None))?; let sql = find_sql(&args)?; diff --git a/crates/persisting-pchronicle-cli/src/server/acceleration.rs b/crates/persisting-pchronicle-cli/src/server/acceleration.rs index 343110bd..e9476a58 100644 --- a/crates/persisting-pchronicle-cli/src/server/acceleration.rs +++ b/crates/persisting-pchronicle-cli/src/server/acceleration.rs @@ -1401,7 +1401,7 @@ mod tests { ) .await?, ); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; let acceleration = ServerAcceleration::default(); let sql = "SELECT _file_, event_id FROM events WHERE agent_id = 'project-a' AND event_id = 'event-a'"; let routed = acceleration.route_sql(&snapshot, &engine, sql).await; diff --git a/crates/persisting-pchronicle-cli/src/server/mod.rs b/crates/persisting-pchronicle-cli/src/server/mod.rs index 04ace98f..f15565f1 100644 --- a/crates/persisting-pchronicle-cli/src/server/mod.rs +++ b/crates/persisting-pchronicle-cli/src/server/mod.rs @@ -224,7 +224,7 @@ async fn build_catalog_runtime( ) .await?, ); - let engine = Arc::new(ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?); + let engine = Arc::new(snapshot.clone().query_engine(Default::default()).await?); Ok(Arc::new(CatalogRuntime { snapshot, engine, diff --git a/crates/persisting-pchronicle-cli/src/tests.rs b/crates/persisting-pchronicle-cli/src/tests.rs index 8c222366..d10061ff 100644 --- a/crates/persisting-pchronicle-cli/src/tests.rs +++ b/crates/persisting-pchronicle-cli/src/tests.rs @@ -1533,7 +1533,7 @@ upstream = "http://{upstream_addr}/v1" ) .await?, ); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot).await?; + let engine = snapshot.query_engine(Default::default()).await?; let rows = engine .query_jsonl( "SELECT kind, COUNT(*) AS count FROM dataset.events GROUP BY kind ORDER BY kind", diff --git a/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs b/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs index 40f2752d..0c394a75 100644 --- a/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs +++ b/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs @@ -15,9 +15,10 @@ use anyhow::{Context, Result}; use async_trait::async_trait; use chrono::{Duration as ChronoDuration, TimeZone, Utc}; use persisting_pchronicle::{ - raw_event_lance_path, CatalogSnapshotOptions, ChronicleQueryEngine, DatasetCatalogSnapshot, - DatasetMount, EventIdentity, EventRecord, LanceMaintenanceOptions, RawEventLanceAppender, - RawEventLanceStore, StoryCoords, + raw_event_lance_path, CatalogSnapshotOptions, ChronicleQueryEngine, + ChronicleQueryExecutionOptions, DatasetCatalogSnapshot, DatasetMount, DocumentFormat, + EventIdentity, EventRecord, LanceMaintenanceOptions, RawEventLanceAppender, RawEventLanceStore, + StoryCoords, }; use serde::Serialize; use serde_json::{json, Value}; @@ -189,7 +190,7 @@ impl PChronicleBackend { .iter() .map(|dataset| dataset.ready_source_count()) .sum(); - self.engine = Some(ChronicleQueryEngine::from_catalog_snapshot(snapshot).await?); + self.engine = Some(snapshot.query_engine(Default::default()).await?); Ok(started.elapsed()) } } @@ -433,16 +434,20 @@ async fn main() -> Result<()> { let rss_after_unpruned_queries = current_rss_bytes(); let fresh_global_point_setup_started = Instant::now(); - let fresh_global_point_engine = - ChronicleQueryEngine::from_catalog_snapshot(discover_catalog(&storage).await?).await?; + let fresh_global_point_engine = discover_catalog(&storage) + .await? + .query_engine(Default::default()) + .await?; let fresh_global_point_setup_ms = duration_ms(fresh_global_point_setup_started.elapsed()); let fresh_global_point_metric = measure_query(&fresh_global_point_engine, &point_sql, 7).await?; drop(fresh_global_point_engine); let fresh_global_list_setup_started = Instant::now(); - let fresh_global_list_engine = - ChronicleQueryEngine::from_catalog_snapshot(discover_catalog(&storage).await?).await?; + let fresh_global_list_engine = discover_catalog(&storage) + .await? + .query_engine(Default::default()) + .await?; let fresh_global_list_setup_ms = duration_ms(fresh_global_list_setup_started.elapsed()); let fresh_global_list_metric = measure_query(&fresh_global_list_engine, &list_sql, 7).await?; drop(fresh_global_list_engine); @@ -452,9 +457,12 @@ async fn main() -> Result<()> { .find(|row| row.logical_id == point_id) .context("fixture contains no point-query row")?; let direct_open_started = Instant::now(); - let direct_engine = - ChronicleQueryEngine::open_events(raw_event_lance_path(&point_row.coords(&storage))?) - .await?; + let direct_engine = ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + raw_event_lance_path(&point_row.coords(&storage))?, + ChronicleQueryExecutionOptions::default(), + ) + .await?; let direct_open_ms = duration_ms(direct_open_started.elapsed()); let direct_point_metric = measure_query( &direct_engine, @@ -469,8 +477,10 @@ async fn main() -> Result<()> { drop(direct_engine); let exact_file_setup_started = Instant::now(); - let exact_file_engine = - ChronicleQueryEngine::from_catalog_snapshot(discover_catalog(&storage).await?).await?; + let exact_file_engine = discover_catalog(&storage) + .await? + .query_engine(Default::default()) + .await?; let exact_file_setup_ms = duration_ms(exact_file_setup_started.elapsed()); let exact_file = format!( "{}/{}/events.lance", @@ -492,8 +502,10 @@ async fn main() -> Result<()> { drop(exact_file_engine); let project_file_setup_started = Instant::now(); - let project_file_engine = - ChronicleQueryEngine::from_catalog_snapshot(discover_catalog(&storage).await?).await?; + let project_file_engine = discover_catalog(&storage) + .await? + .query_engine(Default::default()) + .await?; let project_file_setup_ms = duration_ms(project_file_setup_started.elapsed()); let project_file_list_metric = measure_query( &project_file_engine, @@ -527,7 +539,7 @@ async fn main() -> Result<()> { let base_count = health.physical_rows; let pinned_snapshot = discover_catalog(&storage).await?; - let pinned_engine = ChronicleQueryEngine::from_catalog_snapshot(pinned_snapshot).await?; + let pinned_engine = pinned_snapshot.query_engine(Default::default()).await?; let load_rows = generate_load_rows(&config); let load_query = async { let mut query_samples = Vec::new(); @@ -563,7 +575,12 @@ async fn main() -> Result<()> { ack_samples.push(duration_ms(started.elapsed())); let visibility_started = Instant::now(); let path = raw_event_lance_path(&row.coords(&storage))?; - let fresh = ChronicleQueryEngine::open_events(path).await?; + let fresh = ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + path, + ChronicleQueryExecutionOptions::default(), + ) + .await?; let visible = count_query(&fresh, "SELECT COUNT(*) AS row_count FROM events").await?; anyhow::ensure!( visible == (index + 1) as u64, diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index dda7d40c..c9f70299 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -135,19 +135,20 @@ pub use store::{ AttemptRegistry, CatalogDataset, CatalogErrorPolicy, CatalogNamespace, CatalogPage, CatalogProjectionStatus, CatalogSnapshotOptions, CatalogSourceDescription, CatalogSourceKind, CatalogSourceRevision, CatalogSourceStatus, CatalogStorylineKey, CatalogTrajectoryBundle, - ChronicleQueryBackend, ChronicleQueryEngine, ChronicleQueryExecutionOptions, CommitRunOutcome, - DatasetCatalogSnapshot, DatasetMount, DiscoveredSource, EventFactSnapshot, EventLogLayoutStats, - EventRow, EventWriterFence, ExportOutcome, ExternalTableFormat, ExternalTableSpec, + ChronicleQueryEngine, ChronicleQueryExecutionOptions, CommitRunOutcome, DatasetCatalogSnapshot, + DatasetMount, DiscoveredSource, EventFactSnapshot, EventLogLayoutStats, EventRow, + EventWriterFence, ExportOutcome, ExternalTableFormat, ExternalTableSpec, FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, FileTrajectoryFormat, FileTrajectoryQueryMetrics, FileTrajectoryQueryMetricsSnapshot, LanceMaintenanceOptions, LanceMaintenanceReport, LeaseAcquireOutcome, LocalQueryInputFile, LocalQueryManifest, - LocalQueryManifestOptions, NamespacePath, ProjectionSourceSnapshot, RawEventDataSource, - RawEventDataSourceOptions, RawEventLanceAppender, RawEventLanceStore, RawEventTableProvider, - ReplayOutcome, RunControlStore, StorylineContentOptions, StorylineContentReadMode, - StorylineDataFusionTableNames, StorylineDataSource, StorylineDataSourceOptions, - StorylineLanceStore, StorylineMaintenanceReport, StorylineProjectionLineage, - StorylineStreamImportReport, StorylineTableKind, StorylineTablePaths, StorylineTableProvider, - TrajectoryStats, CATALOG_SOURCES_TABLE, CATALOG_TRAJECTORIES_TABLE, DATAFUSION_EVENTS_TABLE, + LocalQueryManifestOptions, NamespacePath, ProjectionSourceSnapshot, QueryBackendInfo, + QuerySnapshot, RawEventDataSource, RawEventDataSourceOptions, RawEventLanceAppender, + RawEventLanceStore, RawEventTableProvider, ReplayOutcome, RunControlStore, + StorylineContentOptions, StorylineContentReadMode, StorylineDataFusionTableNames, + StorylineDataSource, StorylineDataSourceOptions, StorylineLanceStore, + StorylineMaintenanceReport, StorylineProjectionLineage, StorylineStreamImportReport, + StorylineTableKind, StorylineTablePaths, StorylineTableProvider, TrajectoryStats, + CATALOG_SOURCES_TABLE, CATALOG_TRAJECTORIES_TABLE, DATAFUSION_EVENTS_TABLE, DATAFUSION_RUNS_TABLE, DATAFUSION_STEPS_TABLE, DATAFUSION_TOOL_CALLS_TABLE, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, DEFAULT_DATASET_NAME, DEFAULT_LOCAL_QUERY_BATCH_SIZE, DEFAULT_LOCAL_QUERY_CACHE_BYTES, diff --git a/crates/persisting-pchronicle/src/store/catalog/mod.rs b/crates/persisting-pchronicle/src/store/catalog/mod.rs index 30ed8b3f..fbc839ee 100644 --- a/crates/persisting-pchronicle/src/store/catalog/mod.rs +++ b/crates/persisting-pchronicle/src/store/catalog/mod.rs @@ -213,6 +213,14 @@ pub struct DatasetCatalogSnapshot { } impl DatasetCatalogSnapshot { + /// Build a read-only query engine over this catalog snapshot. + pub async fn query_engine( + self: Arc, + options: super::ChronicleQueryExecutionOptions, + ) -> Result { + super::ChronicleQueryEngine::from_catalog_snapshot_with_options(self, options).await + } + pub async fn discover( mounts: Vec, default_dataset: Option, diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index abe48e8d..ce603416 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -1,8 +1,8 @@ use super::*; use crate::{ build_storyline_projection, rebuild_storyline_projection, story_lance_event_path, - sync_storyline_projection, ChronicleQueryEngine, EventIdentity, RawEventLanceStore, - StoryCoords, StorylineAgent, StorylineLanceStore, StorylineProjectionSyncMode, StorylineTurn, + sync_storyline_projection, EventIdentity, RawEventLanceStore, StoryCoords, StorylineAgent, + StorylineLanceStore, StorylineProjectionSyncMode, StorylineTurn, }; use object_store::ObjectStoreExt; @@ -230,7 +230,7 @@ async fn report_mode_keeps_late_local_format_errors_lazy() -> Result<()> { 0 ); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; let error = engine .query("SELECT run_id FROM dataset.runs WHERE _file_ = 'broken.json'") .await @@ -257,7 +257,7 @@ async fn empty_dataset_still_exposes_the_stable_catalog_tables() -> Result<()> { .await?, ); assert_eq!(snapshot.datasets()[0].sources.len(), 0); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot).await?; + let engine = snapshot.query_engine(Default::default()).await?; let output = engine .query_jsonl("SELECT COUNT(*) AS runs FROM runs") .await?; @@ -283,7 +283,7 @@ async fn catalog_prunes_file_sources_before_lazy_resolution() -> Result<()> { .iter() .all(|source| source.resolution_count.load(Ordering::Relaxed) == 0)); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; assert!(snapshot.prepared[0] .sources .iter() @@ -377,7 +377,7 @@ async fn catalog_downloads_only_selected_remote_file_source() -> Result<()> { .sources .iter() .all(|source| source.resolution_count.load(Ordering::Relaxed) == 0)); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; let rows = engine .query_jsonl("SELECT run_id FROM dataset.runs WHERE _file_ = 'one.json'") .await?; @@ -429,7 +429,7 @@ async fn catalog_prunes_storyline_sources_before_opening_lance() -> Result<()> { .await? .replace_storyline(&storyline("session-a-new", "run-a-new")) .await?; - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; assert!(snapshot.prepared[0] .sources .iter() @@ -507,7 +507,7 @@ async fn one_source_keeps_storylines_with_a_shared_run_id_independent() -> Resul ) .await?, ); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; let rows = engine .query_jsonl( @@ -568,7 +568,7 @@ async fn catalog_joins_require_file_keys_only_within_one_dataset() -> Result<()> ) .await?, ); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot).await?; + let engine = snapshot.query_engine(Default::default()).await?; let unsafe_join = engine .query( @@ -687,7 +687,7 @@ async fn canonical_event_source_exposes_and_loads_each_storyline_independently() .await?; let lazy = &snapshot.prepared[0].sources[0]; assert_eq!(lazy.resolution_count.load(Ordering::Relaxed), 0); - let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot.clone()).await?; + let engine = snapshot.clone().query_engine(Default::default()).await?; assert_eq!(lazy.resolution_count.load(Ordering::Relaxed), 0); let event_count = engine .query_jsonl( @@ -737,7 +737,10 @@ async fn canonical_event_source_exposes_and_loads_each_storyline_independently() stale_snapshot.datasets()[0].sources[0].projection_status, Some(CatalogProjectionStatus::Stale) ); - let stale_engine = ChronicleQueryEngine::from_catalog_snapshot(stale_snapshot.clone()).await?; + let stale_engine = stale_snapshot + .clone() + .query_engine(Default::default()) + .await?; let stale_resolved = stale_snapshot.prepared[0].sources[0].resolve().await?; let ResolvedSource::Events(stale_events) = stale_resolved.as_ref() else { panic!("stale projection did not fall back to canonical events"); @@ -775,7 +778,7 @@ async fn canonical_event_source_exposes_and_loads_each_storyline_independently() ) .await?, ); - let limited_engine = ChronicleQueryEngine::from_catalog_snapshot(limited_snapshot).await?; + let limited_engine = limited_snapshot.query_engine(Default::default()).await?; let limit_error = limited_engine .query("SELECT * FROM dataset.runs") .await diff --git a/crates/persisting-pchronicle/src/store/document_source.rs b/crates/persisting-pchronicle/src/store/document_source.rs index f0b87a1f..7ec96585 100644 --- a/crates/persisting-pchronicle/src/store/document_source.rs +++ b/crates/persisting-pchronicle/src/store/document_source.rs @@ -17,8 +17,8 @@ use crate::{ use super::{ AgenticMdDataSource, AtifReader, FileTrajectoryDataSource, FileTrajectoryFormat, - LocalQueryManifest, RawEventDataSource, StorylineDataSource, StorylineLanceStore, - DEFAULT_MAX_EVENT_FALLBACK_BYTES, DEFAULT_MAX_EVENT_FALLBACK_ROWS, + LocalQueryManifest, RawEventDataSource, StorylineDataSource, DEFAULT_MAX_EVENT_FALLBACK_BYTES, + DEFAULT_MAX_EVENT_FALLBACK_ROWS, }; pub(crate) trait QueryDocumentSource { @@ -37,7 +37,6 @@ pub(crate) enum DocumentSourceImpl { Storyline { path: PathBuf, source: StorylineDataSource, - store: StorylineLanceStore, }, AgenticMd { path: PathBuf, @@ -64,17 +63,7 @@ pub(crate) async fn open_document_source( }), DocumentFormat::Storyline => { let source = StorylineDataSource::open(&path).await.map_err(other)?; - let root = path - .to_str() - .ok_or_else(|| Error::Other("Storyline path is not valid UTF-8".into()))?; - let store = StorylineLanceStore::open_uri_unchecked(root) - .await - .map_err(other)?; - Ok(DocumentSourceImpl::Storyline { - path, - source, - store, - }) + Ok(DocumentSourceImpl::Storyline { path, source }) } DocumentFormat::AgenticMd => { let input = std::fs::read_to_string(&path).map_err(Error::from)?; @@ -170,7 +159,7 @@ impl DocumentSourceImpl { Self::Files { format, manifest, .. } => for_each_file_storyline(*format, manifest, on_storyline), - Self::Storyline { source, store, .. } => { + Self::Storyline { source, .. } => { let context = SessionContext::new(); source.register(&context).map_err(other)?; for session_id in distinct_strings( @@ -180,12 +169,7 @@ impl DocumentSourceImpl { ) .await? { - let story = store - .get_storyline_full(&session_id) - .await - .map_err(other)? - .ok_or_else(|| Error::SessionNotFound(session_id.clone()))?; - on_storyline(story)?; + on_storyline(read_pinned_storyline(&context, &session_id).await?)?; } Ok(()) } @@ -234,6 +218,20 @@ impl DocumentSourceImpl { _ => None, } } + + pub(crate) fn event_snapshot(&self) -> Option<&super::EventFactSnapshot> { + match self { + Self::Events { source, .. } => Some(source.fact_snapshot()), + _ => None, + } + } + + pub(crate) fn storyline_generation(&self) -> Option<&str> { + match self { + Self::Storyline { source, .. } => Some(source.generation()), + _ => None, + } + } } impl QueryDocumentSource for DocumentSourceImpl { @@ -384,6 +382,64 @@ async fn distinct_strings( Ok(values) } +async fn read_pinned_storyline( + context: &SessionContext, + session_id: &str, +) -> Result { + let literal = session_id.replace('\'', "''"); + let runs = context + .sql(&format!( + "SELECT * FROM runs WHERE session_id = '{literal}'" + )) + .await + .map_err(other)? + .collect() + .await + .map_err(other)?; + let steps = context + .sql(&format!( + "SELECT * FROM steps WHERE session_id = '{literal}' ORDER BY step_id" + )) + .await + .map_err(other)? + .collect() + .await + .map_err(other)?; + let tool_calls = context + .sql(&format!( + "SELECT * FROM tool_calls WHERE session_id = '{literal}' ORDER BY step_id, call_index" + )) + .await + .map_err(other)? + .collect() + .await + .map_err(other)?; + + let mut run_rows = Vec::new(); + for batch in &runs { + run_rows.extend(super::story_runs_from_batch(batch).map_err(other)?); + } + if run_rows.len() != 1 { + return Err(Error::Other(format!( + "pinned Storyline source returned {} run rows for session_id '{session_id}'", + run_rows.len() + ))); + } + let mut step_rows = Vec::new(); + for batch in &steps { + step_rows.extend(super::story_steps_from_batch(batch).map_err(other)?); + } + let mut tool_call_rows = Vec::new(); + for batch in &tool_calls { + tool_call_rows.extend(super::story_tool_calls_from_batch(batch).map_err(other)?); + } + crate::reconstruct_storyline(crate::StorylineTables { + run: run_rows.remove(0), + steps: step_rows, + tool_calls: tool_call_rows, + }) +} + fn story_rows(story: &StorylineDocument) -> usize { 1usize.saturating_add(story.turns.len()).saturating_add( story diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 005f7288..8479cbc0 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -92,8 +92,8 @@ pub use local_query_manifest::{ }; #[cfg(feature = "lance-store")] pub use query_engine::{ - ChronicleQueryBackend, ChronicleQueryEngine, ChronicleQueryExecutionOptions, - ExternalTableFormat, ExternalTableSpec, + ChronicleQueryEngine, ChronicleQueryExecutionOptions, ExternalTableFormat, ExternalTableSpec, + QueryBackendInfo, QuerySnapshot, }; #[cfg(feature = "lance-store")] pub use run_control::{CommitRunOutcome, LeaseAcquireOutcome, RunControlStore}; diff --git a/crates/persisting-pchronicle/src/store/query_engine.rs b/crates/persisting-pchronicle/src/store/query_engine.rs index c0a58e52..cb81b413 100644 --- a/crates/persisting-pchronicle/src/store/query_engine.rs +++ b/crates/persisting-pchronicle/src/store/query_engine.rs @@ -17,37 +17,31 @@ use datafusion::sql::sqlparser::ast::Statement as SqlStatement; use futures::TryStreamExt; use super::{ - AtifDataSource, DatasetCatalogSnapshot, FileTrajectoryDataSource, - FileTrajectoryDataSourceOptions, FileTrajectoryFormat, FileTrajectoryQueryMetrics, - FileTrajectoryQueryMetricsSnapshot, LocalQueryManifest, RawEventDataSource, - StorylineDataSource, SOURCE_FILE_COLUMN, + DatasetCatalogSnapshot, FileTrajectoryQueryMetrics, FileTrajectoryQueryMetricsSnapshot, + SOURCE_FILE_COLUMN, }; +use crate::{DocumentFormat, QueryCapabilities, QueryTables}; #[derive(Debug, Clone, PartialEq, Eq)] -pub enum ChronicleQueryBackend { - Catalog { - snapshot_id: String, - datasets: usize, - sources: usize, +pub struct QueryBackendInfo { + pub format: DocumentFormat, + pub tables: QueryTables, + pub capabilities: QueryCapabilities, + pub source_count: usize, + pub snapshot: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum QuerySnapshot { + CanonicalEvent { + format_version: u32, + fact_version: u64, + fact_rows: u64, + layout_revision: u64, }, - Lance { + Storyline { generation: String, }, - Events { - version: u64, - }, - Atif { - files: usize, - documents: Option, - steps: Option, - tool_calls: Option, - }, - OpenaiMsg { - files: usize, - }, - Actf { - files: usize, - }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -93,7 +87,7 @@ impl ExternalTableSpec { /// Read-only SQL engine exposing the same normalized tables for all sources. pub struct ChronicleQueryEngine { context: SessionContext, - backend: ChronicleQueryBackend, + backend_info: Option, require_file_join_key: bool, local_file_metrics: Vec, // Keeps pinned remote-file materializations alive for the complete query. @@ -104,210 +98,67 @@ impl std::fmt::Debug for ChronicleQueryEngine { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("ChronicleQueryEngine") - .field("backend", &self.backend) + .field("backend_info", &self.backend_info) .finish_non_exhaustive() } } impl ChronicleQueryEngine { - pub async fn from_catalog_snapshot(snapshot: Arc) -> Result { - Self::from_catalog_snapshot_with_options( - snapshot, - ChronicleQueryExecutionOptions::default(), - ) - .await - } - - pub async fn from_catalog_snapshot_with_options( - snapshot: Arc, + pub async fn open( + format: DocumentFormat, + path: impl AsRef, options: ChronicleQueryExecutionOptions, ) -> Result { + let source = crate::document::open_document(format, path.as_ref()).await?; let context = query_session_context(&options)?; - snapshot.register(&context).await?; - let datasets = snapshot.datasets().len(); - let sources = snapshot - .datasets() - .iter() - .map(|dataset| dataset.ready_source_count()) - .sum(); - let require_file_join_key = snapshot.requires_file_join_key(); - let snapshot_id = snapshot.snapshot_id().to_string(); - Ok(Self { - context, - backend: ChronicleQueryBackend::Catalog { - snapshot_id, - datasets, - sources, - }, - require_file_join_key, - local_file_metrics: Vec::new(), - _catalog_snapshot: Some(snapshot), - }) - } - - pub async fn open_lance(root: impl AsRef) -> Result { - let source = StorylineDataSource::open(root).await?; - Self::from_lance_source(source) - } - - /// Open a Lance store from a local path or object-store URI such as S3. - pub async fn open_lance_uri(root: impl AsRef) -> Result { - Self::open_lance_uri_with_options(root, ChronicleQueryExecutionOptions::default()).await - } - - pub async fn open_lance_uri_with_options( - root: impl AsRef, - options: ChronicleQueryExecutionOptions, - ) -> Result { - let source = StorylineDataSource::open_uri(root).await?; - Self::from_lance_source_with_options(source, options) - } - - /// Open one canonical fenced `events.lance` manifest as the SQL table `events`. - pub async fn open_events(path: impl AsRef) -> Result { - let source = RawEventDataSource::open(path).await?; - Self::from_events_source(source) - } - - pub async fn open_events_uri(uri: impl AsRef) -> Result { - Self::open_events_uri_with_options(uri, ChronicleQueryExecutionOptions::default()).await - } - - pub async fn open_events_uri_with_options( - uri: impl AsRef, - options: ChronicleQueryExecutionOptions, - ) -> Result { - let source = RawEventDataSource::open_uri(uri).await?; - Self::from_events_source_with_options(source, options) - } - - pub fn open_atif(path: impl AsRef) -> Result { - Self::from_file_trajectory_source(FileTrajectoryDataSource::open_atif(path)?) - } - - pub fn open_openai_msg(path: impl AsRef) -> Result { - Self::from_file_trajectory_source(FileTrajectoryDataSource::open_openai_msg(path)?) - } - - pub fn open_actf(path: impl AsRef) -> Result { - Self::from_file_trajectory_source(FileTrajectoryDataSource::open_actf(path)?) - } - - pub fn open_local_manifest(manifest: LocalQueryManifest) -> Result { - Self::open_local_manifest_with_options(manifest, FileTrajectoryDataSourceOptions::default()) - } - - pub fn open_local_manifest_with_options( - manifest: LocalQueryManifest, - options: FileTrajectoryDataSourceOptions, - ) -> Result { - Self::open_local_manifest_with_execution_options( - manifest, - options, - ChronicleQueryExecutionOptions::default(), - ) - } - - pub fn open_local_manifest_with_execution_options( - manifest: LocalQueryManifest, - file_options: FileTrajectoryDataSourceOptions, - execution_options: ChronicleQueryExecutionOptions, - ) -> Result { - Self::from_file_trajectory_source_with_options( - FileTrajectoryDataSource::from_manifest_with_options(manifest, file_options)?, - execution_options, - ) - } - - pub fn from_lance_source(source: StorylineDataSource) -> Result { - Self::from_lance_source_with_options(source, ChronicleQueryExecutionOptions::default()) - } - - pub fn from_lance_source_with_options( - source: StorylineDataSource, - options: ChronicleQueryExecutionOptions, - ) -> Result { - let generation = source.generation().to_string(); - let context = query_session_context(&options)?; - source.register(&context)?; - Ok(Self { - context, - backend: ChronicleQueryBackend::Lance { generation }, - require_file_join_key: false, - local_file_metrics: Vec::new(), - _catalog_snapshot: None, - }) - } - - pub fn from_events_source(source: RawEventDataSource) -> Result { - Self::from_events_source_with_options(source, ChronicleQueryExecutionOptions::default()) - } - - pub fn from_events_source_with_options( - source: RawEventDataSource, - options: ChronicleQueryExecutionOptions, - ) -> Result { - let version = source.version(); - let context = query_session_context(&options)?; - source.register(&context)?; - Ok(Self { - context, - backend: ChronicleQueryBackend::Events { version }, - require_file_join_key: false, - local_file_metrics: Vec::new(), - _catalog_snapshot: None, - }) - } - - pub fn from_atif_source(source: AtifDataSource) -> Result { - let files = source.file_count(); - let backend = ChronicleQueryBackend::Atif { - files, - documents: source.document_count(), - steps: source.step_count(), - tool_calls: source.tool_call_count(), + let tables = source.register_datafusion(&context)?; + let capabilities = source.capabilities(); + let source_count = source.inner.source_count(); + let snapshot = if let Some(snapshot) = source.inner.event_snapshot() { + Some(QuerySnapshot::CanonicalEvent { + // Existing canonical manifests predate explicit format versioning. + format_version: 1, + fact_version: snapshot.fact_version, + fact_rows: snapshot.fact_rows, + layout_revision: snapshot.layout_revision, + }) + } else { + source + .inner + .storyline_generation() + .map(|generation| QuerySnapshot::Storyline { + generation: generation.to_string(), + }) }; - let context = source.session_context()?; + let local_file_metrics = source.inner.file_metrics().into_iter().collect(); Ok(Self { context, - backend, - require_file_join_key: files > 1, - local_file_metrics: Vec::new(), + backend_info: Some(QueryBackendInfo { + format, + tables, + capabilities, + source_count, + snapshot, + }), + require_file_join_key: source_count > 1, + local_file_metrics, _catalog_snapshot: None, }) } - pub fn from_file_trajectory_source(source: FileTrajectoryDataSource) -> Result { - Self::from_file_trajectory_source_with_options( - source, - ChronicleQueryExecutionOptions::default(), - ) - } - - pub fn from_file_trajectory_source_with_options( - source: FileTrajectoryDataSource, + pub(crate) async fn from_catalog_snapshot_with_options( + snapshot: Arc, options: ChronicleQueryExecutionOptions, ) -> Result { - let files = source.file_count(); - let metrics = source.metrics(); - let backend = match source.format() { - FileTrajectoryFormat::Atif => ChronicleQueryBackend::Atif { - files, - documents: None, - steps: None, - tool_calls: None, - }, - FileTrajectoryFormat::OpenaiMsg => ChronicleQueryBackend::OpenaiMsg { files }, - FileTrajectoryFormat::Actf => ChronicleQueryBackend::Actf { files }, - }; let context = query_session_context(&options)?; - source.register(&context)?; + snapshot.register(&context).await?; + let require_file_join_key = snapshot.requires_file_join_key(); Ok(Self { context, - backend, - require_file_join_key: files > 1, - local_file_metrics: vec![metrics], - _catalog_snapshot: None, + backend_info: None, + require_file_join_key, + local_file_metrics: Vec::new(), + _catalog_snapshot: Some(snapshot), }) } @@ -315,8 +166,8 @@ impl ChronicleQueryEngine { &self.context } - pub fn backend(&self) -> &ChronicleQueryBackend { - &self.backend + pub fn backend_info(&self) -> Option<&QueryBackendInfo> { + self.backend_info.as_ref() } pub fn local_file_metrics(&self) -> Option { diff --git a/crates/persisting-pchronicle/tests/direct_file_query.rs b/crates/persisting-pchronicle/tests/direct_file_query.rs index 07878006..383675c2 100644 --- a/crates/persisting-pchronicle/tests/direct_file_query.rs +++ b/crates/persisting-pchronicle/tests/direct_file_query.rs @@ -4,13 +4,15 @@ use std::fs; use std::path::PathBuf; use anyhow::Result; +use datafusion::prelude::SessionContext; use persisting_pchronicle::detect_local_query_manifest; use persisting_pchronicle::store::{ story_runs_arrow_schema, story_steps_arrow_schema, story_tool_calls_arrow_schema, }; use persisting_pchronicle::{ - ChronicleFormat, ChronicleQueryBackend, ChronicleQueryEngine, FileTrajectoryDataSourceOptions, - LocalQueryManifest, SOURCE_FILE_COLUMN, + ChronicleFormat, ChronicleQueryEngine, ChronicleQueryExecutionOptions, DocumentFormat, + FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, LocalQueryManifest, + SOURCE_FILE_COLUMN, }; fn fixtures() -> PathBuf { @@ -32,11 +34,19 @@ fn json_rows(output: &str) -> Result> { #[tokio::test] async fn queries_one_openai_json_with_the_virtual_file_column() -> Result<()> { let input = fixtures().join("cybergym_0729001_trimmed.json"); - let engine = ChronicleQueryEngine::open_openai_msg(&input)?; - assert!(matches!( - engine.backend(), - ChronicleQueryBackend::OpenaiMsg { files: 1 } - )); + let engine = ChronicleQueryEngine::open( + DocumentFormat::OpenaiMsg, + &input, + ChronicleQueryExecutionOptions::default(), + ) + .await?; + assert_eq!( + engine + .backend_info() + .expect("document backend info") + .source_count, + 1 + ); let rows = json_rows( &engine @@ -70,7 +80,12 @@ async fn auto_detects_and_queries_response_only_openai_rows() -> Result<()> { )?; let manifest = detect_local_query_manifest(temp.path())?; assert_eq!(manifest.format(), ChronicleFormat::OpenaiMsg); - let engine = ChronicleQueryEngine::open_local_manifest(manifest)?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::OpenaiMsg, + manifest.input(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let rows = json_rows(&engine.query_jsonl("SELECT session_id FROM runs").await?)?; assert_eq!(rows[0]["session_id"], "response-only"); Ok(()) @@ -90,11 +105,19 @@ async fn openai_directory_uses_relative_paths_for_like_narrowing() -> Result<()> nested.join("second.json"), )?; - let engine = ChronicleQueryEngine::open_openai_msg(temp.path())?; - assert!(matches!( - engine.backend(), - ChronicleQueryBackend::OpenaiMsg { files: 2 } - )); + let engine = ChronicleQueryEngine::open( + DocumentFormat::OpenaiMsg, + temp.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; + assert_eq!( + engine + .backend_info() + .expect("document backend info") + .source_count, + 2 + ); let rows = json_rows( &engine .query_jsonl( @@ -127,11 +150,19 @@ async fn actf_directory_can_be_narrowed_by_filename_wildcard() -> Result<()> { temp.path().join("protein.actf.json"), )?; - let engine = ChronicleQueryEngine::open_actf(temp.path())?; - assert!(matches!( - engine.backend(), - ChronicleQueryBackend::Actf { files: 2 } - )); + let engine = ChronicleQueryEngine::open( + DocumentFormat::Actf, + temp.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; + assert_eq!( + engine + .backend_info() + .expect("document backend info") + .source_count, + 2 + ); let rows = json_rows( &engine .query_jsonl( @@ -154,7 +185,12 @@ async fn file_like_filter_prunes_unmatched_files_before_they_are_opened() -> Res )?; fs::write(temp.path().join("unmatched.json"), "not-json\n")?; - let engine = ChronicleQueryEngine::open_openai_msg(temp.path())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::OpenaiMsg, + temp.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let rows = json_rows( &engine .query_jsonl( @@ -197,10 +233,18 @@ async fn detected_manifest_is_the_exact_reader_file_set() -> Result<()> { let manifest = detect_local_query_manifest(temp.path())?; fs::write(temp.path().join("added-after-detection.json"), "not-json\n")?; - let engine = ChronicleQueryEngine::open_local_manifest(manifest)?; - let rows = json_rows(&engine.query_jsonl("SELECT session_id FROM runs").await?)?; - assert_eq!(rows.len(), 1); - assert_eq!(rows[0]["session_id"], "cyber-a"); + let source = FileTrajectoryDataSource::from_manifest(manifest)?; + let context = SessionContext::new(); + source.register(&context)?; + let batches = context + .sql("SELECT session_id FROM runs") + .await? + .collect() + .await?; + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); Ok(()) } @@ -223,7 +267,7 @@ async fn shared_cache_reuses_one_normalization_across_virtual_tables() -> Result let input = temp.path().join("input.json"); fs::copy(fixtures().join("cybergym_07270003_trimmed.json"), &input)?; let manifest = LocalQueryManifest::for_format(&input, ChronicleFormat::OpenaiMsg)?; - let engine = ChronicleQueryEngine::open_local_manifest_with_options( + let source = FileTrajectoryDataSource::from_manifest_with_options( manifest, FileTrajectoryDataSourceOptions { cache_files: 1, @@ -231,14 +275,28 @@ async fn shared_cache_reuses_one_normalization_across_virtual_tables() -> Result ..FileTrajectoryDataSourceOptions::default() }, )?; + let context = SessionContext::new(); + source.register(&context)?; assert_eq!( - json_rows(&engine.query_jsonl("SELECT * FROM runs").await?)?.len(), + context + .sql("SELECT * FROM runs") + .await? + .collect() + .await? + .iter() + .map(|batch| batch.num_rows()) + .sum::(), 1 ); fs::write(&input, "not-json")?; - assert!(!engine.query_jsonl("SELECT * FROM steps").await?.is_empty()); - let metrics = engine.local_file_metrics().expect("local metrics"); + assert!(!context + .sql("SELECT * FROM steps") + .await? + .collect() + .await? + .is_empty()); + let metrics = source.metrics().snapshot(); assert_eq!(metrics.files_parsed, 1); assert!(metrics.cache_hits >= 1); assert!(metrics.source_bytes_read > 0); @@ -252,22 +310,36 @@ async fn manifest_fingerprint_and_file_size_limits_fail_closed() -> Result<()> { fs::copy(fixtures().join("cybergym_07270003_trimmed.json"), &input)?; let manifest = LocalQueryManifest::for_format(&input, ChronicleFormat::OpenaiMsg)?; fs::write(&input, "not-json")?; - let engine = ChronicleQueryEngine::open_local_manifest(manifest)?; - let changed = engine.query("SELECT * FROM runs").await.unwrap_err(); + let source = FileTrajectoryDataSource::from_manifest(manifest)?; + let context = SessionContext::new(); + source.register(&context)?; + let changed = context + .sql("SELECT * FROM runs") + .await? + .collect() + .await + .unwrap_err(); assert!(format!("{changed:#}").contains("changed after manifest")); let manifest = LocalQueryManifest::for_format( fixtures().join("cybergym_07270003_trimmed.json"), ChronicleFormat::OpenaiMsg, )?; - let engine = ChronicleQueryEngine::open_local_manifest_with_options( + let source = FileTrajectoryDataSource::from_manifest_with_options( manifest, FileTrajectoryDataSourceOptions { max_file_bytes: 1, ..FileTrajectoryDataSourceOptions::default() }, )?; - let oversized = engine.query("SELECT * FROM runs").await.unwrap_err(); + let context = SessionContext::new(); + source.register(&context)?; + let oversized = context + .sql("SELECT * FROM runs") + .await? + .collect() + .await + .unwrap_err(); assert!(format!("{oversized:#}").contains("max_file_bytes")); Ok(()) } @@ -283,7 +355,12 @@ async fn multi_file_joins_require_the_file_key() -> Result<()> { fixtures().join("cybergym_07270003_trimmed.json"), temp.path().join("two.json"), )?; - let engine = ChronicleQueryEngine::open_openai_msg(temp.path())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::OpenaiMsg, + temp.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let unsafe_join = engine .query( "SELECT * FROM steps s JOIN tool_calls t \ @@ -346,8 +423,18 @@ async fn atif_steps_projection_matches_full_normalization_and_prunes_rows() -> R serde_json::to_vec_pretty(&vec![first, second])?, )?; - let projected = ChronicleQueryEngine::open_atif(&ndjson)?; - let full = ChronicleQueryEngine::open_atif(&compatibility)?; + let projected = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &ndjson, + ChronicleQueryExecutionOptions::default(), + ) + .await?; + let full = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &compatibility, + ChronicleQueryExecutionOptions::default(), + ) + .await?; let queries = [ "SELECT COUNT(*) AS steps FROM steps", "SELECT source, COUNT(*) AS steps FROM steps GROUP BY source ORDER BY source", @@ -412,12 +499,27 @@ async fn projected_atif_streams_ndjson_pretty_object_and_pretty_array() -> Resul fs::write(&array, serde_json::to_vec_pretty(&documents)?)?; let sql = "SELECT session_id, step_id, source FROM steps ORDER BY session_id, step_id"; - let ndjson_engine = ChronicleQueryEngine::open_atif(&ndjson)?; + let ndjson_engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &ndjson, + ChronicleQueryExecutionOptions::default(), + ) + .await?; let ndjson_rows = ndjson_engine.query_jsonl(sql).await?; - let array_engine = ChronicleQueryEngine::open_atif(&array)?; + let array_engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &array, + ChronicleQueryExecutionOptions::default(), + ) + .await?; assert_eq!(array_engine.query_jsonl(sql).await?, ndjson_rows); - let object_engine = ChronicleQueryEngine::open_atif(&object)?; + let object_engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &object, + ChronicleQueryExecutionOptions::default(), + ) + .await?; let object_rows = object_engine.query_jsonl(sql).await?; assert_eq!(json_rows(&object_rows)?.len(), 10); assert_eq!(json_rows(&ndjson_rows)?.len(), 170); @@ -453,7 +555,12 @@ async fn projected_atif_ignores_unselected_large_values_without_materializing_th let temp = tempfile::NamedTempFile::with_suffix(".json")?; fs::write(temp.path(), serde_json::to_vec_pretty(&trajectory)?)?; - let engine = ChronicleQueryEngine::open_atif(temp.path())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + temp.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let rows = json_rows( &engine .query_jsonl("SELECT step_id FROM steps ORDER BY step_id") @@ -474,15 +581,19 @@ async fn projected_ndjson_rejects_a_record_above_the_configured_bound() -> Resul let temp = tempfile::NamedTempFile::with_suffix(".ndjson")?; fs::write(temp.path(), format!("{}\n", serde_json::to_string(&value)?))?; let manifest = LocalQueryManifest::for_format(temp.path(), ChronicleFormat::Atif)?; - let engine = ChronicleQueryEngine::open_local_manifest_with_options( + let source = FileTrajectoryDataSource::from_manifest_with_options( manifest, FileTrajectoryDataSourceOptions { max_record_bytes: 512, ..FileTrajectoryDataSourceOptions::default() }, )?; - let error = engine - .query("SELECT COUNT(*) FROM steps") + let context = SessionContext::new(); + source.register(&context)?; + let error = context + .sql("SELECT COUNT(*) FROM steps") + .await? + .collect() .await .unwrap_err(); assert!( @@ -502,15 +613,19 @@ async fn projected_array_enforces_record_bounds_and_json_separators() -> Result< let oversized = tempfile::NamedTempFile::with_suffix(".json")?; fs::write(oversized.path(), format!("[{record}]"))?; let manifest = LocalQueryManifest::for_format(oversized.path(), ChronicleFormat::Atif)?; - let engine = ChronicleQueryEngine::open_local_manifest_with_options( + let source = FileTrajectoryDataSource::from_manifest_with_options( manifest, FileTrajectoryDataSourceOptions { max_record_bytes: 512, ..FileTrajectoryDataSourceOptions::default() }, )?; - let error = engine - .query("SELECT COUNT(*) FROM steps") + let context = SessionContext::new(); + source.register(&context)?; + let error = context + .sql("SELECT COUNT(*) FROM steps") + .await? + .collect() .await .unwrap_err(); assert!( @@ -520,7 +635,12 @@ async fn projected_array_enforces_record_bounds_and_json_separators() -> Result< let missing_comma = tempfile::NamedTempFile::with_suffix(".json")?; fs::write(missing_comma.path(), format!("[{record} {record}]"))?; - let engine = ChronicleQueryEngine::open_atif(missing_comma.path())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + missing_comma.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let error = engine .query("SELECT COUNT(*) FROM steps") .await @@ -529,7 +649,12 @@ async fn projected_array_enforces_record_bounds_and_json_separators() -> Result< let trailing_comma = tempfile::NamedTempFile::with_suffix(".json")?; fs::write(trailing_comma.path(), format!("[{record},]"))?; - let engine = ChronicleQueryEngine::open_atif(trailing_comma.path())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + trailing_comma.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let error = engine .query("SELECT COUNT(*) FROM steps") .await diff --git a/crates/persisting-pchronicle/tests/production_scale.rs b/crates/persisting-pchronicle/tests/production_scale.rs index 615973dd..1ce5b507 100644 --- a/crates/persisting-pchronicle/tests/production_scale.rs +++ b/crates/persisting-pchronicle/tests/production_scale.rs @@ -18,9 +18,9 @@ use std::time::Instant; use anyhow::{Context, Result}; use persisting_agentctl::RunId; use persisting_pchronicle::{ - raw_event_lance_path, ChronicleQueryEngine, EventIdentity, EventRecord, EventWriterFence, - LanceMaintenanceOptions, LeaseAcquireOutcome, RawEventLanceAppender, RawEventLanceStore, - RunControlStore, StoryCoords, + raw_event_lance_path, ChronicleQueryEngine, ChronicleQueryExecutionOptions, DocumentFormat, + EventIdentity, EventRecord, EventWriterFence, LanceMaintenanceOptions, LeaseAcquireOutcome, + RawEventLanceAppender, RawEventLanceStore, RunControlStore, StoryCoords, }; const CI_STORIES: usize = 4; @@ -228,7 +228,12 @@ async fn event_queries_pin_a_snapshot_while_append_continues() -> Result<()> { append_batches(&sessions, 0, half, 64).await?; let path = raw_event_lance_path(&session)?; - let pinned = ChronicleQueryEngine::open_events(&path).await?; + let pinned = ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + &path, + ChronicleQueryExecutionOptions::default(), + ) + .await?; append_batches(&sessions, half, half, 64).await?; let pinned_count = count_from_jsonl( @@ -238,14 +243,19 @@ async fn event_queries_pin_a_snapshot_while_append_continues() -> Result<()> { )?; assert_eq!(pinned_count, (half * 64) as u64); - let current = ChronicleQueryEngine::open_events(&path).await?; + let current = ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + &path, + ChronicleQueryExecutionOptions::default(), + ) + .await?; let current_count = count_from_jsonl( ¤t .query_jsonl("SELECT COUNT(*) AS row_count FROM events") .await?, )?; assert_eq!(current_count, (half * 2 * 64) as u64); - assert_ne!(pinned.backend(), current.backend()); + assert_ne!(pinned.backend_info(), current.backend_info()); Ok(()) } diff --git a/crates/persisting-pchronicle/tests/query_engine.rs b/crates/persisting-pchronicle/tests/query_engine.rs index 90231b04..e60e75b1 100644 --- a/crates/persisting-pchronicle/tests/query_engine.rs +++ b/crates/persisting-pchronicle/tests/query_engine.rs @@ -6,10 +6,9 @@ use anyhow::Result; use datafusion::prelude::SessionContext; use persisting_pchronicle::{ into_storyline, AtifDataSource, AtifDataSourceOptions, AtifReader, AtifTrajectory, - ChronicleFormat, ChronicleQueryBackend, ChronicleQueryEngine, ChronicleQueryExecutionOptions, - EventIdentity, EventRecord, ExternalTableFormat, ExternalTableSpec, - FileTrajectoryDataSourceOptions, LocalQueryManifest, RawEventLanceStore, StoryCoords, - StorylineDataFusionTableNames, StorylineLanceStore, + ChronicleFormat, ChronicleQueryEngine, ChronicleQueryExecutionOptions, DocumentFormat, + EventIdentity, EventRecord, ExternalTableFormat, ExternalTableSpec, QuerySnapshot, QueryTables, + RawEventLanceStore, StoryCoords, StorylineDataFusionTableNames, StorylineLanceStore, }; const SHARED_SQL: &str = @@ -52,6 +51,27 @@ fn write_ndjson(path: &Path, trajectories: &[AtifTrajectory]) -> Result<()> { Ok(()) } +#[tokio::test] +async fn unified_open_reports_backend_capabilities() -> Result<()> { + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + fixture_root(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; + let backend = engine.backend_info().expect("document backend info"); + assert_eq!(backend.format, DocumentFormat::Atif); + assert_eq!(backend.tables, QueryTables::Storyline); + assert!(backend.capabilities.streaming_decode); + assert_eq!(backend.source_count, 8); + assert_eq!(backend.snapshot, None); + assert!(!matches!( + backend.snapshot, + Some(QuerySnapshot::Storyline { .. }) + )); + Ok(()) +} + #[test] fn atif_datasource_accepts_json_array_jsonl_and_directory() -> Result<()> { let trajectories = load_trajectories()?; @@ -120,7 +140,12 @@ async fn atif_file_filter_prunes_before_validation_and_exposes_relative_path() - temp.path().join("good.json"), )?; std::fs::write(temp.path().join("unmatched.json"), "not-json")?; - let engine = ChronicleQueryEngine::open_atif(temp.path())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + temp.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let output = engine .query_jsonl("SELECT _file_ FROM runs WHERE _file_ = 'good.json'") .await?; @@ -255,17 +280,19 @@ async fn atif_datasource_validates_inputs_and_custom_table_names() -> Result<()> #[tokio::test] async fn same_sql_returns_identical_results_for_lance_and_atif() -> Result<()> { let trajectories = load_trajectories()?; - let atif_engine = - ChronicleQueryEngine::from_atif_source(AtifDataSource::from_trajectories(&trajectories)?)?; - assert!(matches!( - atif_engine.backend(), - ChronicleQueryBackend::Atif { - files: 0, - documents: Some(8), - steps: Some(118), - tool_calls: Some(23) - } - )); + let atif_dir = tempfile::tempdir()?; + let atif_path = atif_dir.path().join("input.ndjson"); + write_ndjson(&atif_path, &trajectories)?; + let atif_engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &atif_path, + ChronicleQueryExecutionOptions::default(), + ) + .await?; + let atif_backend = atif_engine.backend_info().expect("document backend info"); + assert_eq!(atif_backend.format, DocumentFormat::Atif); + assert_eq!(atif_backend.source_count, 1); + assert_eq!(atif_backend.snapshot, None); let dir = tempfile::tempdir()?; let store = StorylineLanceStore::open(dir.path()).await?; @@ -279,10 +306,17 @@ async fn same_sql_returns_identical_results_for_lance_and_atif() -> Result<()> { }) .collect::>>()?; store.replace_storylines(&stories).await?; - let lance_engine = ChronicleQueryEngine::open_lance(dir.path()).await?; + let lance_engine = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + dir.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; assert!(matches!( - lance_engine.backend(), - ChronicleQueryBackend::Lance { .. } + lance_engine + .backend_info() + .and_then(|backend| backend.snapshot.as_ref()), + Some(QuerySnapshot::Storyline { .. }) )); let atif_jsonl = atif_engine.query_jsonl(SHARED_SQL).await?; @@ -316,8 +350,18 @@ async fn timestamp_milliseconds_match_for_lance_and_direct_atif_queries() -> Res .collect::>>()?; store.replace_storylines(&stories).await?; - let lance = ChronicleQueryEngine::open_lance(dir.path()).await?; - let atif = ChronicleQueryEngine::open_atif(fixture_root())?; + let lance = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + dir.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; + let atif = ChronicleQueryEngine::open( + DocumentFormat::Atif, + fixture_root(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let sql = "SELECT session_id, step_id, timestamp \ FROM steps \ WHERE timestamp >= TIMESTAMP '2026-06-15T09:00:10Z' \ @@ -333,9 +377,14 @@ async fn timestamp_milliseconds_match_for_lance_and_direct_atif_queries() -> Res #[tokio::test] async fn streaming_jsonl_matches_collected_jsonl() -> Result<()> { let trajectories = load_trajectories()?; - let engine = ChronicleQueryEngine::from_atif_source(AtifDataSource::from_trajectories( - &trajectories[..1], - )?)?; + let input = tempfile::NamedTempFile::with_suffix(".json")?; + std::fs::write(input.path(), serde_json::to_vec(&trajectories[..1])?)?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + input.path(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let sql = "SELECT session_id, step_id FROM steps ORDER BY step_id"; let collected = engine.query_jsonl(sql).await?; let mut streamed = Vec::new(); @@ -354,28 +403,29 @@ async fn streaming_jsonl_matches_collected_jsonl() -> Result<()> { #[tokio::test] async fn query_runtime_validates_memory_and_spill_limits() -> Result<()> { let input = fixture_root().join("dialogue_10.json"); - let manifest = LocalQueryManifest::for_format(&input, ChronicleFormat::Atif)?; - let invalid = ChronicleQueryEngine::open_local_manifest_with_execution_options( - manifest.clone(), - FileTrajectoryDataSourceOptions::default(), + let invalid = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &input, ChronicleQueryExecutionOptions { memory_limit_bytes: Some(0), ..ChronicleQueryExecutionOptions::default() }, ) + .await .unwrap_err(); assert!(invalid.to_string().contains("memory_limit_bytes")); let spill = tempfile::tempdir()?; - let engine = ChronicleQueryEngine::open_local_manifest_with_execution_options( - manifest, - FileTrajectoryDataSourceOptions::default(), + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + &input, ChronicleQueryExecutionOptions { memory_limit_bytes: Some(64 * 1024 * 1024), spill_path: Some(spill.path().to_path_buf()), max_spill_bytes: Some(256 * 1024 * 1024), }, - )?; + ) + .await?; assert!(!engine .query_jsonl("SELECT COUNT(*) FROM steps") .await? @@ -385,7 +435,12 @@ async fn query_runtime_validates_memory_and_spill_limits() -> Result<()> { #[tokio::test] async fn query_engine_joins_csv_and_json_external_tables() -> Result<()> { - let engine = ChronicleQueryEngine::open_atif(fixture_root())?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + fixture_root(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let temp = tempfile::tempdir()?; let labels = temp.path().join("labels.csv"); std::fs::write( @@ -489,7 +544,12 @@ async fn query_engine_opens_object_store_uri() -> Result<()> { .replace_storylines(&stories) .await?; - let engine = ChronicleQueryEngine::open_lance_uri(&uri).await?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + Path::new(&uri), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let output = engine .query_jsonl("SELECT COUNT(*) AS runs FROM runs") .await?; @@ -500,7 +560,7 @@ async fn query_engine_opens_object_store_uri() -> Result<()> { // An engine pins one immutable version tuple. Moving CURRENT must not change // the result of an already planned federated/long-running query. - let pinned_generation = engine.backend().clone(); + let pinned_generation = engine.backend_info().cloned(); StorylineLanceStore::open_uri(&uri) .await? .replace_storyline(&into_storyline( @@ -515,9 +575,14 @@ async fn query_engine_opens_object_store_uri() -> Result<()> { serde_json::from_str::(pinned_output.trim())?["runs"], 2 ); - assert_eq!(engine.backend(), &pinned_generation); + assert_eq!(engine.backend_info(), pinned_generation.as_ref()); - let reopened = ChronicleQueryEngine::open_lance_uri(&uri).await?; + let reopened = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + Path::new(&uri), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let current_output = reopened .query_jsonl("SELECT COUNT(*) AS runs FROM runs") .await?; @@ -525,7 +590,7 @@ async fn query_engine_opens_object_store_uri() -> Result<()> { serde_json::from_str::(current_output.trim())?["runs"], 3 ); - assert_ne!(reopened.backend(), &pinned_generation); + assert_ne!(reopened.backend_info(), pinned_generation.as_ref()); Ok(()) } @@ -538,9 +603,13 @@ async fn query_engine_rejects_empty_object_store_without_current() -> Result<()> .duration_since(std::time::UNIX_EPOCH)? .as_nanos() ); - let error = ChronicleQueryEngine::open_lance_uri(&uri) - .await - .unwrap_err(); + let error = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + Path::new(&uri), + ChronicleQueryExecutionOptions::default(), + ) + .await + .unwrap_err(); assert!( error.to_string().contains("no committed generation"), "{error:#}" @@ -579,10 +648,22 @@ async fn query_engine_exposes_canonical_events_table() -> Result<()> { RawEventLanceStore.append_events(&session, &records).await?; let path = persisting_pchronicle::raw_event_lance_path(&session)?; - let engine = ChronicleQueryEngine::open_events(&path).await?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + &path, + ChronicleQueryExecutionOptions::default(), + ) + .await?; assert!(matches!( - engine.backend(), - ChronicleQueryBackend::Events { version } if *version > 0 + engine + .backend_info() + .and_then(|backend| backend.snapshot.as_ref()), + Some(QuerySnapshot::CanonicalEvent { + format_version: 1, + fact_version, + fact_rows: 2, + layout_revision, + }) if *fact_version > 0 && *layout_revision > 0 )); let output = engine .query_jsonl("SELECT seq, session_id, kind, payload_json FROM events ORDER BY seq") @@ -600,36 +681,44 @@ async fn query_engine_exposes_canonical_events_table() -> Result<()> { Ok(()) } -#[test] -fn query_engine_rejects_writes_and_multiple_statements() -> Result<()> { - let engine = ChronicleQueryEngine::open_atif(fixture_root())?; - let runtime = tokio::runtime::Builder::new_current_thread().build()?; +#[tokio::test] +async fn query_engine_rejects_writes_and_multiple_statements() -> Result<()> { + let engine = ChronicleQueryEngine::open( + DocumentFormat::Atif, + fixture_root(), + ChronicleQueryExecutionOptions::default(), + ) + .await?; - let copy_error = runtime - .block_on(engine.dataframe("COPY steps TO '/tmp/pchronicle.parquet' STORED AS PARQUET")) + let copy_error = engine + .dataframe("COPY steps TO '/tmp/pchronicle.parquet' STORED AS PARQUET") + .await .expect_err("COPY must be rejected"); assert!(copy_error.to_string().contains("only accepts")); - let multi_error = runtime - .block_on(engine.dataframe("SELECT 1; SELECT 2")) + let multi_error = engine + .dataframe("SELECT 1; SELECT 2") + .await .expect_err("multiple statements must be rejected"); assert!(multi_error.to_string().contains("exactly one")); - let empty_error = runtime - .block_on(engine.dataframe("")) + let empty_error = engine + .dataframe("") + .await .expect_err("empty SQL must be rejected"); assert!(empty_error.to_string().contains("exactly one")); - let insert_error = runtime - .block_on(engine.dataframe("INSERT INTO steps VALUES (1)")) + let insert_error = engine + .dataframe("INSERT INTO steps VALUES (1)") + .await .expect_err("INSERT must be rejected"); assert!(insert_error.to_string().contains("only accepts")); - let values = runtime.block_on(engine.query_jsonl("VALUES (1), (2)"))?; + let values = engine.query_jsonl("VALUES (1), (2)").await?; assert_eq!(values.lines().count(), 2); - let explain = runtime.block_on(engine.query("EXPLAIN SELECT * FROM runs"))?; + let explain = engine.query("EXPLAIN SELECT * FROM runs").await?; assert!(!explain.is_empty()); - let empty_result = runtime.block_on(engine.query_jsonl("SELECT * FROM runs WHERE 1 = 0"))?; + let empty_result = engine.query_jsonl("SELECT * FROM runs WHERE 1 = 0").await?; assert!(empty_result.is_empty()); Ok(()) } diff --git a/crates/persisting-pchronicle/tests/s3_storage.rs b/crates/persisting-pchronicle/tests/s3_storage.rs index 50643028..329d35a1 100644 --- a/crates/persisting-pchronicle/tests/s3_storage.rs +++ b/crates/persisting-pchronicle/tests/s3_storage.rs @@ -6,9 +6,9 @@ use anyhow::{Context, Result}; use lance::io::ObjectStore; use persisting_pchronicle::{ - into_storyline, AtifTrajectory, ChronicleFormat, ChronicleQueryEngine, EventRecord, - LanceMaintenanceOptions, RawEventLanceAppender, RawEventLanceStore, StoryCoords, - StorylineLanceStore, + into_storyline, AtifTrajectory, ChronicleFormat, ChronicleQueryEngine, + ChronicleQueryExecutionOptions, DocumentFormat, EventRecord, LanceMaintenanceOptions, + RawEventLanceAppender, RawEventLanceStore, StoryCoords, StorylineLanceStore, }; fn unique_root() -> Result { @@ -104,7 +104,12 @@ async fn run_contract(root: &str) -> Result<()> { let store = StorylineLanceStore::open_uri(&storyline_root).await?; let first = fixture_storyline()?; store.replace_storyline(&first).await?; - let pinned = ChronicleQueryEngine::open_lance_uri(&storyline_root).await?; + let pinned = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + std::path::Path::new(&storyline_root), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let mut second = first.clone(); second.session_id = "s3-contract-second".into(); @@ -125,7 +130,12 @@ async fn run_contract(root: &str) -> Result<()> { .await?, [Some(first), Some(second)] ); - let engine = ChronicleQueryEngine::open_lance_uri(&storyline_root).await?; + let engine = ChronicleQueryEngine::open( + DocumentFormat::Storyline, + std::path::Path::new(&storyline_root), + ChronicleQueryExecutionOptions::default(), + ) + .await?; let output = engine .query_jsonl("SELECT COUNT(*) AS runs FROM runs") .await?; @@ -161,8 +171,10 @@ async fn run_append_scale_contract(root: &str) -> Result<()> { writer.append_event_batch(&entries).await?; if batch_index + 1 == BATCHES / 2 { pinned = Some( - ChronicleQueryEngine::open_events_uri( - persisting_pchronicle::raw_event_lance_path(&session)?.to_string_lossy(), + ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + persisting_pchronicle::raw_event_lance_path(&session)?, + ChronicleQueryExecutionOptions::default(), ) .await?, ); @@ -179,8 +191,10 @@ async fn run_append_scale_contract(root: &str) -> Result<()> { (BATCHES * ROWS_PER_BATCH / 2) as u64 ); - let current = ChronicleQueryEngine::open_events_uri( - persisting_pchronicle::raw_event_lance_path(&session)?.to_string_lossy(), + let current = ChronicleQueryEngine::open( + DocumentFormat::CanonicalEvent, + persisting_pchronicle::raw_event_lance_path(&session)?, + ChronicleQueryExecutionOptions::default(), ) .await?; let current_output = current From 5a33198081b55a7e46aa68877575eb6ca7632eec Mon Sep 17 00:00:00 2001 From: Reiase Date: Tue, 18 Aug 2026 00:38:10 +0800 Subject: [PATCH 26/65] refactor: remove legacy atif data source --- .../benches/lance_vs_json.rs | 21 +- crates/persisting-pchronicle/src/lib.rs | 35 +- .../src/store/atif_datafusion.rs | 544 ------------------ .../src/store/files/atif_reader.rs | 186 ++++++ .../src/store/files/mod.rs | 23 +- crates/persisting-pchronicle/src/store/mod.rs | 15 +- .../src/store/storyline/mod.rs | 2 +- .../tests/query_engine.rs | 101 ++-- 8 files changed, 279 insertions(+), 648 deletions(-) delete mode 100644 crates/persisting-pchronicle/src/store/atif_datafusion.rs create mode 100644 crates/persisting-pchronicle/src/store/files/atif_reader.rs diff --git a/crates/persisting-pchronicle/benches/lance_vs_json.rs b/crates/persisting-pchronicle/benches/lance_vs_json.rs index 62750518..8273a6cb 100644 --- a/crates/persisting-pchronicle/benches/lance_vs_json.rs +++ b/crates/persisting-pchronicle/benches/lance_vs_json.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use persisting_pchronicle::{ - from_storyline, into_storyline, AtifDataSource, AtifTrajectory, ChronicleFormat, + from_storyline, into_storyline, AtifTrajectory, ChronicleFormat, FileTrajectoryDataSource, StorylineDataSource, StorylineDocument, StorylineLanceStore, }; @@ -73,7 +73,7 @@ fn main() -> Result<()> { struct Comparison { lance: Duration, - atif_datafusion: Duration, + atif_file: Duration, json_scan: Duration, json_memory: Duration, } @@ -127,7 +127,7 @@ async fn run(scale: usize, iterations: usize) -> Result { std::fs::write(&json_path, json_document.as_bytes())?; let json_write = json_write_started.elapsed(); let atif_open_started = Instant::now(); - let atif_source = AtifDataSource::open(&json_path)?; + let atif_source = FileTrajectoryDataSource::open_atif(&json_path)?; let atif_open = atif_open_started.elapsed(); let atif_context = atif_source.session_context()?; let parsed = atif_lines @@ -191,7 +191,7 @@ async fn run(scale: usize, iterations: usize) -> Result { let selective = Comparison { lance: time_async_query(&selective_query, iterations).await?, - atif_datafusion: time_async_query(&atif_selective_query, iterations).await?, + atif_file: time_async_query(&atif_selective_query, iterations).await?, json_scan: time_sync(iterations, || { json_selective_query(&json_path, &target_session) })?, @@ -201,7 +201,7 @@ async fn run(scale: usize, iterations: usize) -> Result { }; let analytical = Comparison { lance: time_async_query(&analytical_query, iterations).await?, - atif_datafusion: time_async_query(&atif_analytical_query, iterations).await?, + atif_file: time_async_query(&atif_analytical_query, iterations).await?, json_scan: time_sync(iterations, || json_analysis(&json_path))?, json_memory: time_sync(iterations, || Ok(json_memory_analysis(&parsed)))?, }; @@ -378,9 +378,8 @@ fn time_sync(iterations: usize, mut operation: impl FnMut() -> Result) -> fn print_comparison(id: &str, name: &str, iterations: usize, comparison: &Comparison) { let lance_qps = iterations as f64 / comparison.lance.as_secs_f64(); - let atif_qps = iterations as f64 / comparison.atif_datafusion.as_secs_f64(); - let atif_over_lance_time = - comparison.atif_datafusion.as_secs_f64() / comparison.lance.as_secs_f64(); + let atif_qps = iterations as f64 / comparison.atif_file.as_secs_f64(); + let atif_over_lance_time = comparison.atif_file.as_secs_f64() / comparison.lance.as_secs_f64(); println!("{name}:"); println!( " Lance/DataFusion indexed: {:?} ({:.1} queries/s)", @@ -388,7 +387,7 @@ fn print_comparison(id: &str, name: &str, iterations: usize, comparison: &Compar ); println!( " ATIF/DataFusion stream: {:?} ({:.1} queries/s, Lance speed ratio {:.2}x)", - comparison.atif_datafusion, atif_qps, atif_over_lance_time + comparison.atif_file, atif_qps, atif_over_lance_time ); println!( " JSON read+Serde scan: {:?} ({:.1} queries/s, Lance {:.2}x faster)", @@ -416,9 +415,9 @@ fn print_conclusion(result: &BenchmarkResult) { let group_disk_speedup = result.analytical.json_scan.as_secs_f64() / result.analytical.lance.as_secs_f64(); let selective_memory_ratio = - result.selective.atif_datafusion.as_secs_f64() / result.selective.lance.as_secs_f64(); + result.selective.atif_file.as_secs_f64() / result.selective.lance.as_secs_f64(); let group_memory_ratio = - result.analytical.atif_datafusion.as_secs_f64() / result.analytical.lance.as_secs_f64(); + result.analytical.atif_file.as_secs_f64() / result.analytical.lance.as_secs_f64(); println!("Conclusion:"); println!( " Storage: Lance uses {:.2}% of JSON space, saving {:.2}%.", diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index c9f70299..e9404806 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -131,24 +131,23 @@ pub use store::{ event_records_from_batch, event_row_from_batch, event_row_to_event_record, event_rows_from_batch, event_rows_to_batch, export_source_dirs, export_story_bundle, load_atif_trajectories, raw_event_arrow_schema, raw_event_lance_path, AppendOutcome, - AtifDataSource, AtifDataSourceOptions, AtifReader, AttemptRecord, AttemptRecordState, - AttemptRegistry, CatalogDataset, CatalogErrorPolicy, CatalogNamespace, CatalogPage, - CatalogProjectionStatus, CatalogSnapshotOptions, CatalogSourceDescription, CatalogSourceKind, - CatalogSourceRevision, CatalogSourceStatus, CatalogStorylineKey, CatalogTrajectoryBundle, - ChronicleQueryEngine, ChronicleQueryExecutionOptions, CommitRunOutcome, DatasetCatalogSnapshot, - DatasetMount, DiscoveredSource, EventFactSnapshot, EventLogLayoutStats, EventRow, - EventWriterFence, ExportOutcome, ExternalTableFormat, ExternalTableSpec, - FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, FileTrajectoryFormat, - FileTrajectoryQueryMetrics, FileTrajectoryQueryMetricsSnapshot, LanceMaintenanceOptions, - LanceMaintenanceReport, LeaseAcquireOutcome, LocalQueryInputFile, LocalQueryManifest, - LocalQueryManifestOptions, NamespacePath, ProjectionSourceSnapshot, QueryBackendInfo, - QuerySnapshot, RawEventDataSource, RawEventDataSourceOptions, RawEventLanceAppender, - RawEventLanceStore, RawEventTableProvider, ReplayOutcome, RunControlStore, - StorylineContentOptions, StorylineContentReadMode, StorylineDataFusionTableNames, - StorylineDataSource, StorylineDataSourceOptions, StorylineLanceStore, - StorylineMaintenanceReport, StorylineProjectionLineage, StorylineStreamImportReport, - StorylineTableKind, StorylineTablePaths, StorylineTableProvider, TrajectoryStats, - CATALOG_SOURCES_TABLE, CATALOG_TRAJECTORIES_TABLE, DATAFUSION_EVENTS_TABLE, + AtifReader, AttemptRecord, AttemptRecordState, AttemptRegistry, CatalogDataset, + CatalogErrorPolicy, CatalogNamespace, CatalogPage, CatalogProjectionStatus, + CatalogSnapshotOptions, CatalogSourceDescription, CatalogSourceKind, CatalogSourceRevision, + CatalogSourceStatus, CatalogStorylineKey, CatalogTrajectoryBundle, ChronicleQueryEngine, + ChronicleQueryExecutionOptions, CommitRunOutcome, DatasetCatalogSnapshot, DatasetMount, + DiscoveredSource, EventFactSnapshot, EventLogLayoutStats, EventRow, EventWriterFence, + ExportOutcome, ExternalTableFormat, ExternalTableSpec, FileTrajectoryDataSource, + FileTrajectoryDataSourceOptions, FileTrajectoryFormat, FileTrajectoryQueryMetrics, + FileTrajectoryQueryMetricsSnapshot, LanceMaintenanceOptions, LanceMaintenanceReport, + LeaseAcquireOutcome, LocalQueryInputFile, LocalQueryManifest, LocalQueryManifestOptions, + NamespacePath, ProjectionSourceSnapshot, QueryBackendInfo, QuerySnapshot, RawEventDataSource, + RawEventDataSourceOptions, RawEventLanceAppender, RawEventLanceStore, RawEventTableProvider, + ReplayOutcome, RunControlStore, StorylineContentOptions, StorylineContentReadMode, + StorylineDataFusionTableNames, StorylineDataSource, StorylineDataSourceOptions, + StorylineLanceStore, StorylineMaintenanceReport, StorylineProjectionLineage, + StorylineStreamImportReport, StorylineTableKind, StorylineTablePaths, StorylineTableProvider, + TrajectoryStats, CATALOG_SOURCES_TABLE, CATALOG_TRAJECTORIES_TABLE, DATAFUSION_EVENTS_TABLE, DATAFUSION_RUNS_TABLE, DATAFUSION_STEPS_TABLE, DATAFUSION_TOOL_CALLS_TABLE, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, DEFAULT_DATASET_NAME, DEFAULT_LOCAL_QUERY_BATCH_SIZE, DEFAULT_LOCAL_QUERY_CACHE_BYTES, diff --git a/crates/persisting-pchronicle/src/store/atif_datafusion.rs b/crates/persisting-pchronicle/src/store/atif_datafusion.rs deleted file mode 100644 index db3edd08..00000000 --- a/crates/persisting-pchronicle/src/store/atif_datafusion.rs +++ /dev/null @@ -1,544 +0,0 @@ -//! DataFusion datasource for ATIF JSON and JSONL inputs. -//! -//! File-backed ATIF inputs are validated with a bounded-memory pass and exposed -//! as repeatable DataFusion `StreamingTable`s. Every scan reopens the input, -//! normalizes one trajectory at a time through Storyline, and emits bounded -//! Arrow batches using the same schemas as the three-table Lance store. -//! -//! Explicit in-memory constructors retain `MemTable` behavior because their -//! callers have already materialized the complete input. - -use std::collections::HashSet; -#[cfg(test)] -use std::collections::VecDeque; -use std::fs::File; -use std::io::{BufRead, BufReader, Lines}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use datafusion::datasource::{MemTable, TableProvider}; -use datafusion::prelude::SessionContext; -use lance::deps::arrow_array::RecordBatch; -use lance::deps::arrow_schema::SchemaRef; - -use crate::convert::atif_to_storyline; -use crate::{AtifTrajectory, ChronicleFormat}; -#[cfg(test)] -use crate::{StoryRunRow, StoryStepRow, StoryToolCallRow}; - -#[cfg(test)] -use super::StorylineTableKind; -use super::{ - story_runs_arrow_schema, story_runs_to_batch, story_steps_arrow_schema, story_steps_to_batch, - story_tool_calls_arrow_schema, story_tool_calls_to_batch, FileTrajectoryDataSource, - FileTrajectoryDataSourceOptions, LocalQueryInputFile, LocalQueryManifest, - StorylineDataFusionTableNames, -}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct AtifDataSourceOptions { - /// Maximum rows per Arrow batch/partition. - pub batch_size: usize, -} - -impl Default for AtifDataSourceOptions { - fn default() -> Self { - Self { batch_size: 8192 } - } -} - -#[derive(Debug)] -pub struct AtifDataSource { - runs: Arc, - steps: Arc, - tool_calls: Arc, - known_stats: Option, - file_count: usize, -} - -impl AtifDataSource { - /// Open a single ATIF JSON/JSONL file, a directory of such files, or a - /// directory containing ATIF JSON, JSONL, or NDJSON documents. - pub fn open(path: impl AsRef) -> Result { - Self::open_with_options(path, AtifDataSourceOptions::default()) - } - - pub fn open_with_options( - path: impl AsRef, - options: AtifDataSourceOptions, - ) -> Result { - let manifest = LocalQueryManifest::for_format(path, ChronicleFormat::Atif)?; - Self::from_manifest_with_options(manifest, options) - } - - pub fn from_manifest(manifest: LocalQueryManifest) -> Result { - Self::from_manifest_with_options(manifest, AtifDataSourceOptions::default()) - } - - pub fn from_manifest_with_options( - manifest: LocalQueryManifest, - options: AtifDataSourceOptions, - ) -> Result { - validate_options(options)?; - anyhow::ensure!( - manifest.format() == ChronicleFormat::Atif, - "ATIF datasource requires an ATIF manifest" - ); - let source = FileTrajectoryDataSource::from_manifest_with_options( - manifest, - FileTrajectoryDataSourceOptions { - batch_size: options.batch_size, - ..FileTrajectoryDataSourceOptions::default() - }, - )?; - let (runs, steps, tool_calls, file_count, _metrics) = source.into_providers(); - Ok(Self { - runs, - steps, - tool_calls, - known_stats: None, - file_count, - }) - } - - /// Parse a single ATIF object, an array of objects, or JSONL/NDJSON. - pub fn from_json(input: &str) -> Result { - Self::from_json_with_options(input, AtifDataSourceOptions::default()) - } - - pub fn from_json_with_options(input: &str, options: AtifDataSourceOptions) -> Result { - validate_options(options)?; - let trajectories = parse_documents(input)?; - Self::from_trajectories_with_options(&trajectories, options) - } - - pub fn from_trajectories(trajectories: &[AtifTrajectory]) -> Result { - Self::from_trajectories_with_options(trajectories, AtifDataSourceOptions::default()) - } - - pub fn from_trajectories_with_options( - trajectories: &[AtifTrajectory], - options: AtifDataSourceOptions, - ) -> Result { - validate_options(options)?; - if trajectories.is_empty() { - anyhow::bail!("ATIF datasource requires at least one trajectory"); - } - - let mut session_ids = HashSet::with_capacity(trajectories.len()); - let mut runs = Vec::with_capacity(trajectories.len()); - let mut steps = Vec::new(); - let mut tool_calls = Vec::new(); - for trajectory in trajectories { - trajectory.validate().map_err(anyhow::Error::from)?; - let story = atif_to_storyline(trajectory).map_err(anyhow::Error::from)?; - let tables = crate::split_storyline(&story).map_err(anyhow::Error::from)?; - if !session_ids.insert(tables.run.session_id.clone()) { - anyhow::bail!("duplicate ATIF session_id '{}'", tables.run.session_id); - } - runs.push(tables.run); - steps.extend(tables.steps); - tool_calls.extend(tables.tool_calls); - } - runs.sort_by(|a, b| a.session_id.cmp(&b.session_id)); - steps.sort_by(|a, b| { - a.session_id - .cmp(&b.session_id) - .then(a.step_id.cmp(&b.step_id)) - }); - tool_calls.sort_by(|a, b| { - a.session_id - .cmp(&b.session_id) - .then(a.step_id.cmp(&b.step_id)) - .then(a.call_index.cmp(&b.call_index)) - }); - - let known_stats = AtifInputStats { - document_count: runs.len(), - step_count: steps.len(), - tool_call_count: tool_calls.len(), - }; - Ok(Self { - runs: Arc::new(mem_table( - story_runs_arrow_schema(), - &runs, - options.batch_size, - story_runs_to_batch, - )?), - steps: Arc::new(mem_table( - story_steps_arrow_schema(), - &steps, - options.batch_size, - story_steps_to_batch, - )?), - tool_calls: Arc::new(mem_table( - story_tool_calls_arrow_schema(), - &tool_calls, - options.batch_size, - story_tool_calls_to_batch, - )?), - known_stats: Some(known_stats), - file_count: 0, - }) - } - - /// Counts are known without I/O only for explicitly in-memory inputs. - pub fn document_count(&self) -> Option { - self.known_stats.as_ref().map(|stats| stats.document_count) - } - - pub fn step_count(&self) -> Option { - self.known_stats.as_ref().map(|stats| stats.step_count) - } - - pub fn tool_call_count(&self) -> Option { - self.known_stats.as_ref().map(|stats| stats.tool_call_count) - } - - pub fn file_count(&self) -> usize { - self.file_count - } - - pub fn register(&self, context: &SessionContext) -> Result<()> { - self.register_as(context, &StorylineDataFusionTableNames::default()) - } - - pub fn register_as( - &self, - context: &SessionContext, - names: &StorylineDataFusionTableNames, - ) -> Result<()> { - validate_table_names(names)?; - register(context, &names.runs, self.runs.clone())?; - register(context, &names.steps, self.steps.clone())?; - register(context, &names.tool_calls, self.tool_calls.clone())?; - Ok(()) - } - - pub fn session_context(&self) -> Result { - let context = SessionContext::new(); - self.register(&context)?; - Ok(context) - } -} - -/// Bounded-memory ATIF reader. -/// -/// JSONL/NDJSON inputs are decoded one non-empty line at a time. Directories -/// are traversed in stable path order and only the current file is open. A -/// regular `.json` file may contain one object or an array and is buffered per -/// file for compatibility; large corpora should use NDJSON. -pub struct AtifReader { - files: std::vec::IntoIter, - current: Option, -} - -enum AtifFileReader { - Lines { - path: PathBuf, - lines: Lines>, - line_number: usize, - }, - Documents(std::vec::IntoIter), -} - -impl AtifReader { - pub fn open(path: impl AsRef) -> Result { - let manifest = LocalQueryManifest::for_format(path, ChronicleFormat::Atif)?; - Ok(Self::from_files(manifest.files())) - } - - pub(crate) fn from_manifest(manifest: &LocalQueryManifest) -> Self { - Self::from_files(manifest.files()) - } - - fn from_files(files: &[LocalQueryInputFile]) -> Self { - Self { - files: files - .iter() - .map(|file| file.path().to_path_buf()) - .collect::>() - .into_iter(), - current: None, - } - } - - fn open_file(path: PathBuf) -> Result { - match path.extension().and_then(|value| value.to_str()) { - Some("jsonl" | "ndjson") => { - let file = File::open(&path) - .with_context(|| format!("open ATIF datasource {}", path.display()))?; - Ok(AtifFileReader::Lines { - path, - lines: BufReader::new(file).lines(), - line_number: 0, - }) - } - _ => { - let input = std::fs::read_to_string(&path) - .with_context(|| format!("read ATIF datasource {}", path.display()))?; - let documents = parse_documents(&input) - .with_context(|| format!("parse ATIF datasource {}", path.display()))?; - Ok(AtifFileReader::Documents(documents.into_iter())) - } - } - } -} - -impl Iterator for AtifReader { - type Item = Result; - - fn next(&mut self) -> Option { - loop { - if let Some(current) = &mut self.current { - match current { - AtifFileReader::Documents(documents) => { - if let Some(document) = documents.next() { - return Some(Ok(document)); - } - } - AtifFileReader::Lines { - path, - lines, - line_number, - } => { - for line in lines.by_ref() { - *line_number += 1; - let line = match line { - Ok(line) => line, - Err(error) => { - return Some(Err(error).with_context(|| { - format!( - "read ATIF datasource {} line {}", - path.display(), - line_number - ) - })); - } - }; - if line.trim().is_empty() { - continue; - } - return Some( - AtifTrajectory::from_json_str(line.trim()) - .map_err(anyhow::Error::from) - .with_context(|| { - format!( - "parse ATIF datasource {} line {}", - path.display(), - line_number - ) - }), - ); - } - } - } - self.current = None; - } - - let path = self.files.next()?; - match Self::open_file(path) { - Ok(reader) => self.current = Some(reader), - Err(error) => return Some(Err(error)), - } - } - } -} - -#[derive(Debug, Default)] -pub(crate) struct AtifInputStats { - pub document_count: usize, - pub step_count: usize, - pub tool_call_count: usize, -} - -#[cfg(test)] -struct AtifBatchIterator { - reader: AtifReader, - kind: StorylineTableKind, - batch_size: usize, - runs: VecDeque, - steps: VecDeque, - tool_calls: VecDeque, - finished: bool, -} - -#[cfg(test)] -impl AtifBatchIterator { - fn new(reader: AtifReader, kind: StorylineTableKind, batch_size: usize) -> Self { - Self { - reader, - kind, - batch_size, - runs: VecDeque::new(), - steps: VecDeque::new(), - tool_calls: VecDeque::new(), - finished: false, - } - } - - fn pending_len(&self) -> usize { - match self.kind { - StorylineTableKind::Runs => self.runs.len(), - StorylineTableKind::Steps => self.steps.len(), - StorylineTableKind::ToolCalls => self.tool_calls.len(), - } - } - - fn push_trajectory(&mut self, trajectory: AtifTrajectory) -> Result<()> { - let story = atif_to_storyline(&trajectory).map_err(anyhow::Error::from)?; - let tables = crate::split_storyline(&story).map_err(anyhow::Error::from)?; - match self.kind { - StorylineTableKind::Runs => self.runs.push_back(tables.run), - StorylineTableKind::Steps => self.steps.extend(tables.steps), - StorylineTableKind::ToolCalls => self.tool_calls.extend(tables.tool_calls), - } - Ok(()) - } - - fn encode_pending(&mut self) -> Result { - let count = self.pending_len().min(self.batch_size); - match self.kind { - StorylineTableKind::Runs => { - let rows = self.runs.drain(..count).collect::>(); - story_runs_to_batch(&rows) - } - StorylineTableKind::Steps => { - let rows = self.steps.drain(..count).collect::>(); - story_steps_to_batch(&rows) - } - StorylineTableKind::ToolCalls => { - let rows = self.tool_calls.drain(..count).collect::>(); - story_tool_calls_to_batch(&rows) - } - } - } -} - -#[cfg(test)] -impl Iterator for AtifBatchIterator { - type Item = Result; - - fn next(&mut self) -> Option { - while !self.finished && self.pending_len() < self.batch_size { - match self.reader.next() { - Some(Ok(trajectory)) => { - if let Err(error) = self.push_trajectory(trajectory) { - self.finished = true; - return Some(Err(error)); - } - } - Some(Err(error)) => { - self.finished = true; - return Some(Err(error)); - } - None => self.finished = true, - } - } - if self.pending_len() > 0 { - Some(self.encode_pending()) - } else { - None - } - } -} - -fn validate_options(options: AtifDataSourceOptions) -> Result<()> { - if options.batch_size == 0 { - anyhow::bail!("ATIF datasource batch_size must be greater than zero"); - } - Ok(()) -} - -fn mem_table( - schema: SchemaRef, - rows: &[T], - batch_size: usize, - encode: fn(&[T]) -> Result, -) -> Result { - let partitions = if rows.is_empty() { - vec![Vec::new()] - } else { - rows.chunks(batch_size) - .map(|chunk| encode(chunk).map(|batch| vec![batch])) - .collect::>>()? - }; - MemTable::try_new(schema, partitions).context("build ATIF DataFusion MemTable") -} - -fn register(context: &SessionContext, name: &str, provider: Arc) -> Result<()> { - context - .register_table(name, provider) - .with_context(|| format!("register ATIF DataFusion table '{name}'"))?; - Ok(()) -} - -fn validate_table_names(names: &StorylineDataFusionTableNames) -> Result<()> { - let values = [&names.runs, &names.steps, &names.tool_calls]; - if values.iter().any(|name| name.trim().is_empty()) { - anyhow::bail!("DataFusion table names must not be empty"); - } - if names.runs == names.steps - || names.runs == names.tool_calls - || names.steps == names.tool_calls - { - anyhow::bail!("DataFusion table names must be distinct"); - } - Ok(()) -} - -/// Load and validate ATIF documents for callers that partition work before -/// building a query engine. -pub fn load_atif_trajectories(path: impl AsRef) -> Result> { - AtifReader::open(path)?.collect() -} - -pub(crate) fn parse_documents(input: &str) -> Result> { - let trimmed = input.trim(); - if trimmed.is_empty() { - anyhow::bail!("ATIF input is empty"); - } - if let Ok(trajectory) = serde_json::from_str::(trimmed) { - trajectory.validate().map_err(anyhow::Error::from)?; - return Ok(vec![trajectory]); - } - if let Ok(trajectories) = serde_json::from_str::>(trimmed) { - for trajectory in &trajectories { - trajectory.validate().map_err(anyhow::Error::from)?; - } - return Ok(trajectories); - } - trimmed - .lines() - .enumerate() - .filter(|(_, line)| !line.trim().is_empty()) - .map(|(index, line)| { - AtifTrajectory::from_json_str(line) - .map_err(anyhow::Error::from) - .with_context(|| format!("parse ATIF JSONL line {}", index + 1)) - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - fn fixture_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/atif") - } - - #[test] - fn streaming_step_batches_respect_the_configured_bound() { - let reader = AtifReader::open(fixture_root()).unwrap(); - let batches = AtifBatchIterator::new(reader, StorylineTableKind::Steps, 3) - .collect::>>() - .unwrap(); - assert!(batches.iter().all(|batch| batch.num_rows() <= 3)); - assert_eq!( - batches.iter().map(RecordBatch::num_rows).sum::(), - 118 - ); - } -} diff --git a/crates/persisting-pchronicle/src/store/files/atif_reader.rs b/crates/persisting-pchronicle/src/store/files/atif_reader.rs new file mode 100644 index 00000000..592ce1d4 --- /dev/null +++ b/crates/persisting-pchronicle/src/store/files/atif_reader.rs @@ -0,0 +1,186 @@ +//! Bounded-memory ATIF document reader shared by conversion and query paths. + +use std::fs::File; +use std::io::{BufRead, BufReader, Lines}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +use crate::{AtifTrajectory, ChronicleFormat}; + +use super::{LocalQueryInputFile, LocalQueryManifest}; + +/// Bounded-memory ATIF reader. +/// +/// JSONL/NDJSON inputs are decoded one non-empty line at a time. Directories +/// are traversed in stable path order and only the current file is open. A +/// regular `.json` file may contain one object or an array and is buffered per +/// file for compatibility; large corpora should use NDJSON. +pub struct AtifReader { + files: std::vec::IntoIter, + current: Option, +} + +enum AtifFileReader { + Lines { + path: PathBuf, + lines: Lines>, + line_number: usize, + }, + Documents(std::vec::IntoIter), +} + +impl AtifReader { + pub fn open(path: impl AsRef) -> Result { + let manifest = LocalQueryManifest::for_format(path, ChronicleFormat::Atif)?; + Ok(Self::from_files(manifest.files())) + } + + pub(crate) fn from_manifest(manifest: &LocalQueryManifest) -> Self { + Self::from_files(manifest.files()) + } + + fn from_files(files: &[LocalQueryInputFile]) -> Self { + Self { + files: files + .iter() + .map(|file| file.path().to_path_buf()) + .collect::>() + .into_iter(), + current: None, + } + } + + fn open_file(path: PathBuf) -> Result { + match path.extension().and_then(|value| value.to_str()) { + Some("jsonl" | "ndjson") => { + let file = File::open(&path) + .with_context(|| format!("open ATIF datasource {}", path.display()))?; + Ok(AtifFileReader::Lines { + path, + lines: BufReader::new(file).lines(), + line_number: 0, + }) + } + _ => { + let input = std::fs::read_to_string(&path) + .with_context(|| format!("read ATIF datasource {}", path.display()))?; + let documents = parse_documents(&input) + .with_context(|| format!("parse ATIF datasource {}", path.display()))?; + Ok(AtifFileReader::Documents(documents.into_iter())) + } + } + } +} + +impl Iterator for AtifReader { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if let Some(current) = &mut self.current { + match current { + AtifFileReader::Documents(documents) => { + if let Some(document) = documents.next() { + return Some(Ok(document)); + } + } + AtifFileReader::Lines { + path, + lines, + line_number, + } => { + for line in lines.by_ref() { + *line_number += 1; + let line = match line { + Ok(line) => line, + Err(error) => { + return Some(Err(error).with_context(|| { + format!( + "read ATIF datasource {} line {}", + path.display(), + line_number + ) + })); + } + }; + if line.trim().is_empty() { + continue; + } + return Some( + AtifTrajectory::from_json_str(line.trim()) + .map_err(anyhow::Error::from) + .with_context(|| { + format!( + "parse ATIF datasource {} line {}", + path.display(), + line_number + ) + }), + ); + } + } + } + self.current = None; + } + + let path = self.files.next()?; + match Self::open_file(path) { + Ok(reader) => self.current = Some(reader), + Err(error) => return Some(Err(error)), + } + } + } +} + +/// Load and validate ATIF documents for callers that partition work before +/// building a query engine. +pub fn load_atif_trajectories(path: impl AsRef) -> Result> { + AtifReader::open(path)?.collect() +} + +pub(crate) fn parse_documents(input: &str) -> Result> { + let trimmed = input.trim(); + if trimmed.is_empty() { + anyhow::bail!("ATIF input is empty"); + } + if let Ok(trajectory) = serde_json::from_str::(trimmed) { + trajectory.validate().map_err(anyhow::Error::from)?; + return Ok(vec![trajectory]); + } + if let Ok(trajectories) = serde_json::from_str::>(trimmed) { + for trajectory in &trajectories { + trajectory.validate().map_err(anyhow::Error::from)?; + } + return Ok(trajectories); + } + trimmed + .lines() + .enumerate() + .filter(|(_, line)| !line.trim().is_empty()) + .map(|(index, line)| { + AtifTrajectory::from_json_str(line) + .map_err(anyhow::Error::from) + .with_context(|| format!("parse ATIF JSONL line {}", index + 1)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_object_array_and_jsonl() { + let object = r#"{"session_id":"s","agent":{"name":"a"},"steps":[]}"#; + assert_eq!(parse_documents(object).unwrap().len(), 1); + assert_eq!(parse_documents(&format!("[{object}]")).unwrap().len(), 1); + assert_eq!( + parse_documents(&format!("{object}\n{object}\n")) + .unwrap() + .len(), + 2 + ); + assert!(parse_documents("").is_err()); + } +} diff --git a/crates/persisting-pchronicle/src/store/files/mod.rs b/crates/persisting-pchronicle/src/store/files/mod.rs index 45d4a858..61c78463 100644 --- a/crates/persisting-pchronicle/src/store/files/mod.rs +++ b/crates/persisting-pchronicle/src/store/files/mod.rs @@ -3,8 +3,11 @@ //! Each source file is one streaming partition. Query-only `_file_` predicates //! are evaluated against the frozen manifest before partitions are opened. +mod atif_reader; mod atif_stream; +pub(crate) use atif_reader::parse_documents as parse_atif_documents; +pub use atif_reader::{load_atif_trajectories, AtifReader}; use atif_stream::stream_projected_atif_steps; use std::collections::{HashMap, HashSet, VecDeque}; @@ -197,14 +200,6 @@ pub struct FileTrajectoryDataSource { metrics: FileTrajectoryQueryMetrics, } -pub(crate) type FileTrajectoryProviderParts = ( - Arc, - Arc, - Arc, - usize, - FileTrajectoryQueryMetrics, -); - impl FileTrajectoryDataSource { pub fn open_openai_msg(path: impl AsRef) -> Result { Self::open(path, FileTrajectoryFormat::OpenaiMsg) @@ -269,16 +264,6 @@ impl FileTrajectoryDataSource { }) } - pub(crate) fn into_providers(self) -> FileTrajectoryProviderParts { - ( - self.runs, - self.steps, - self.tool_calls, - self.file_count, - self.metrics, - ) - } - pub fn format(&self) -> FileTrajectoryFormat { self.format } @@ -897,7 +882,7 @@ fn parse_file( batch_size: usize, ) -> Result { let stories = match format { - FileTrajectoryFormat::Atif => super::atif_datafusion::parse_documents(content) + FileTrajectoryFormat::Atif => parse_atif_documents(content) .with_context(|| format!("parse ATIF input {}", file.path().display()))? .into_iter() .map(|trajectory| atif_to_storyline(&trajectory).map_err(anyhow::Error::from)) diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 8479cbc0..7de7bef0 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -10,8 +10,6 @@ use anyhow::Context as _; #[cfg(feature = "lance-store")] mod agenticmd_datafusion; #[cfg(feature = "lance-store")] -mod atif_datafusion; -#[cfg(feature = "lance-store")] mod attempt_registry; #[cfg(feature = "lance-store")] mod catalog; @@ -45,10 +43,6 @@ mod storyline_model; #[cfg(feature = "lance-store")] pub(crate) use agenticmd_datafusion::AgenticMdDataSource; #[cfg(feature = "lance-store")] -pub use atif_datafusion::{ - load_atif_trajectories, AtifDataSource, AtifDataSourceOptions, AtifReader, -}; -#[cfg(feature = "lance-store")] pub use attempt_registry::{ unix_now_ms as attempt_registry_now_ms, AttemptRecord, AttemptRecordState, AttemptRegistry, }; @@ -79,10 +73,11 @@ pub use events::{ }; #[cfg(feature = "lance-store")] pub use files::{ - FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, FileTrajectoryFormat, - FileTrajectoryQueryMetrics, FileTrajectoryQueryMetricsSnapshot, DEFAULT_LOCAL_QUERY_BATCH_SIZE, - DEFAULT_LOCAL_QUERY_CACHE_BYTES, DEFAULT_LOCAL_QUERY_CACHE_FILES, - DEFAULT_LOCAL_QUERY_MAX_FILE_BYTES, DEFAULT_LOCAL_QUERY_MAX_RECORD_BYTES, SOURCE_FILE_COLUMN, + load_atif_trajectories, AtifReader, FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, + FileTrajectoryFormat, FileTrajectoryQueryMetrics, FileTrajectoryQueryMetricsSnapshot, + DEFAULT_LOCAL_QUERY_BATCH_SIZE, DEFAULT_LOCAL_QUERY_CACHE_BYTES, + DEFAULT_LOCAL_QUERY_CACHE_FILES, DEFAULT_LOCAL_QUERY_MAX_FILE_BYTES, + DEFAULT_LOCAL_QUERY_MAX_RECORD_BYTES, SOURCE_FILE_COLUMN, }; #[cfg(feature = "lance-store")] pub use local_query_manifest::{ diff --git a/crates/persisting-pchronicle/src/store/storyline/mod.rs b/crates/persisting-pchronicle/src/store/storyline/mod.rs index 1446cb93..6837eece 100644 --- a/crates/persisting-pchronicle/src/store/storyline/mod.rs +++ b/crates/persisting-pchronicle/src/store/storyline/mod.rs @@ -80,7 +80,7 @@ use self::content::{ hydrate_batches, open_objects, prune_unreferenced_objects, PendingContent, STORYLINE_OBJECTS_DATASET, }; -use super::atif_datafusion::AtifReader; +use super::AtifReader; use super::{root_write_lock, LanceMaintenanceOptions, LanceMaintenanceReport}; const CURRENT_FILE: &str = "CURRENT"; diff --git a/crates/persisting-pchronicle/tests/query_engine.rs b/crates/persisting-pchronicle/tests/query_engine.rs index e60e75b1..4912787a 100644 --- a/crates/persisting-pchronicle/tests/query_engine.rs +++ b/crates/persisting-pchronicle/tests/query_engine.rs @@ -5,9 +5,10 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use datafusion::prelude::SessionContext; use persisting_pchronicle::{ - into_storyline, AtifDataSource, AtifDataSourceOptions, AtifReader, AtifTrajectory, - ChronicleFormat, ChronicleQueryEngine, ChronicleQueryExecutionOptions, DocumentFormat, - EventIdentity, EventRecord, ExternalTableFormat, ExternalTableSpec, QuerySnapshot, QueryTables, + into_storyline, AtifReader, AtifTrajectory, ChronicleFormat, ChronicleQueryEngine, + ChronicleQueryExecutionOptions, DocumentFormat, EventIdentity, EventRecord, + ExternalTableFormat, ExternalTableSpec, FileTrajectoryDataSource, + FileTrajectoryDataSourceOptions, LocalQueryManifest, QuerySnapshot, QueryTables, RawEventLanceStore, StoryCoords, StorylineDataFusionTableNames, StorylineLanceStore, }; @@ -72,30 +73,36 @@ async fn unified_open_reports_backend_capabilities() -> Result<()> { Ok(()) } -#[test] -fn atif_datasource_accepts_json_array_jsonl_and_directory() -> Result<()> { +#[tokio::test] +async fn generic_atif_source_accepts_json_array_jsonl_and_directory() -> Result<()> { let trajectories = load_trajectories()?; - let array = serde_json::to_string(&trajectories)?; - let from_array = AtifDataSource::from_json(&array)?; - assert_eq!(from_array.document_count(), Some(8)); - assert_eq!(from_array.step_count(), Some(118)); - assert_eq!(from_array.tool_call_count(), Some(23)); - let dir = tempfile::tempdir()?; + let array = dir.path().join("atif-array.json"); + std::fs::write(&array, serde_json::to_vec(&trajectories)?)?; + let from_array = FileTrajectoryDataSource::open_atif(&array)?; + let context = from_array.session_context()?; + let counts = context + .sql("SELECT (SELECT COUNT(*) FROM runs) AS runs, (SELECT COUNT(*) FROM steps) AS steps, (SELECT COUNT(*) FROM tool_calls) AS tool_calls") + .await? + .collect() + .await?; + assert_eq!( + counts.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); + let ndjson = dir.path().join("atif.ndjson"); write_ndjson(&ndjson, &trajectories)?; - let from_jsonl = AtifDataSource::open(&ndjson)?; - assert_eq!(from_jsonl.document_count(), None); - assert_eq!(from_jsonl.step_count(), None); + let from_jsonl = FileTrajectoryDataSource::open_atif(&ndjson)?; + assert_eq!(from_jsonl.file_count(), 1); - let from_directory = AtifDataSource::open(fixture_root())?; - assert_eq!(from_directory.document_count(), None); - assert_eq!(from_directory.step_count(), None); + let from_directory = FileTrajectoryDataSource::open_atif(fixture_root())?; + assert_eq!(from_directory.file_count(), 8); Ok(()) } #[test] -fn atif_datasource_and_reader_share_the_recursive_manifest() -> Result<()> { +fn generic_atif_source_and_reader_share_the_recursive_manifest() -> Result<()> { let temp = tempfile::tempdir()?; let nested = temp.path().join("nested"); std::fs::create_dir(&nested)?; @@ -104,15 +111,15 @@ fn atif_datasource_and_reader_share_the_recursive_manifest() -> Result<()> { nested.join("input.json"), )?; - let source = AtifDataSource::open(temp.path())?; - assert_eq!(source.document_count(), None); + let source = FileTrajectoryDataSource::open_atif(temp.path())?; + assert_eq!(source.file_count(), 1); assert_eq!(AtifReader::open(temp.path())?.count(), 1); Ok(()) } #[tokio::test] async fn default_atif_file_datasource_uses_repeatable_streaming_plan() -> Result<()> { - let source = AtifDataSource::open(fixture_root())?; + let source = FileTrajectoryDataSource::open_atif(fixture_root())?; let context = source.session_context()?; let dataframe = context.sql("SELECT COUNT(*) AS steps FROM steps").await?; let plan = dataframe.clone().create_physical_plan().await?; @@ -186,32 +193,17 @@ fn atif_reader_streams_ndjson_and_directories_in_path_order() -> Result<()> { } #[tokio::test] -async fn atif_datasource_validates_inputs_and_custom_table_names() -> Result<()> { - assert!(AtifDataSource::from_json("").is_err()); - assert!(AtifDataSource::from_json("[]").is_err()); - assert!( - AtifDataSource::from_json_with_options("{}", AtifDataSourceOptions { batch_size: 0 }) - .unwrap_err() - .to_string() - .contains("batch_size") - ); - +async fn generic_atif_source_validates_inputs_and_custom_table_names() -> Result<()> { let trajectories = load_trajectories()?; - let duplicate = vec![trajectories[0].clone(), trajectories[0].clone()]; - assert!(AtifDataSource::from_trajectories(&duplicate) - .unwrap_err() - .to_string() - .contains("duplicate ATIF session_id")); - let dir = tempfile::tempdir()?; - assert!(AtifDataSource::open(dir.path()).is_err()); + assert!(FileTrajectoryDataSource::open_atif(dir.path()).is_err()); let duplicate_jsonl = dir.path().join("duplicate.ndjson"); let duplicate_line = serde_json::to_string(&trajectories[0])?; std::fs::write( &duplicate_jsonl, format!("{duplicate_line}\n{duplicate_line}\n"), )?; - let duplicate_source = AtifDataSource::open(&duplicate_jsonl)?; + let duplicate_source = FileTrajectoryDataSource::open_atif(&duplicate_jsonl)?; let duplicate_error = duplicate_source .session_context()? .sql("SELECT * FROM runs") @@ -222,7 +214,7 @@ async fn atif_datasource_validates_inputs_and_custom_table_names() -> Result<()> assert!(format!("{duplicate_error:#}").contains("duplicate atif session_id")); let invalid_jsonl = dir.path().join("invalid.jsonl"); std::fs::write(&invalid_jsonl, "{}\nnot-json\n")?; - let invalid_source = AtifDataSource::open(&invalid_jsonl)?; + let invalid_source = FileTrajectoryDataSource::open_atif(&invalid_jsonl)?; let invalid_error = invalid_source .session_context()? .sql("SELECT * FROM runs") @@ -235,9 +227,25 @@ async fn atif_datasource_validates_inputs_and_custom_table_names() -> Result<()> "{invalid_error:#}" ); - let source = AtifDataSource::from_trajectories_with_options( - &trajectories[..2], - AtifDataSourceOptions { batch_size: 3 }, + let selected = dir.path().join("selected.json"); + std::fs::write(&selected, serde_json::to_vec(&trajectories[..2])?)?; + let manifest = LocalQueryManifest::for_format(&selected, ChronicleFormat::Atif)?; + assert!(FileTrajectoryDataSource::from_manifest_with_options( + manifest.clone(), + FileTrajectoryDataSourceOptions { + batch_size: 0, + ..FileTrajectoryDataSourceOptions::default() + }, + ) + .unwrap_err() + .to_string() + .contains("batch_size")); + let source = FileTrajectoryDataSource::from_manifest_with_options( + manifest, + FileTrajectoryDataSourceOptions { + batch_size: 3, + ..FileTrajectoryDataSourceOptions::default() + }, )?; let context = SessionContext::new(); source.register_as( @@ -272,8 +280,11 @@ async fn atif_datasource_validates_inputs_and_custom_table_names() -> Result<()> assert!(source.register_as(&context, &empty_name).is_err()); let missing = dir.path().join("missing.json"); - assert!(AtifDataSource::open(missing).is_err()); - assert_eq!(AtifDataSource::open(dir.path())?.file_count(), 2); + assert!(FileTrajectoryDataSource::open_atif(missing).is_err()); + assert_eq!( + FileTrajectoryDataSource::open_atif(dir.path())?.file_count(), + 3 + ); Ok(()) } From bdb984a505ddb8db8b5d772659c98302651156ad Mon Sep 17 00:00:00 2001 From: Reiase Date: Tue, 18 Aug 2026 00:45:52 +0800 Subject: [PATCH 27/65] refactor: use persisting events trajectory protocol directly --- .../persisting-pchronicle-cli/src/control.rs | 65 +- crates/persisting-pchronicle-cli/src/tests.rs | 33 + crates/persisting-pchronicle/src/messages.rs | 126 --- .../src/operations/mod.rs | 3 +- .../src/operations/trajectory/mod.rs | 193 ---- .../src/operations/trajectory/tests.rs | 876 ------------------ 6 files changed, 88 insertions(+), 1208 deletions(-) delete mode 100644 crates/persisting-pchronicle/src/operations/trajectory/mod.rs delete mode 100644 crates/persisting-pchronicle/src/operations/trajectory/tests.rs diff --git a/crates/persisting-pchronicle-cli/src/control.rs b/crates/persisting-pchronicle-cli/src/control.rs index b79bf63b..a1d2155d 100644 --- a/crates/persisting-pchronicle-cli/src/control.rs +++ b/crates/persisting-pchronicle-cli/src/control.rs @@ -5,11 +5,16 @@ use anyhow::{Context, Result}; use persisting_events::{ + AttemptRecord as ProtocolAttemptRecord, AttemptRecordState as ProtocolAttemptRecordState, ChronicleControlEnvelope, ChronicleControlReady, ChronicleControlRequest, ChronicleControlResponse, ChronicleControlResponseEnvelope, CommitRunOutcome, - LeaseAcquireOutcome, CHRONICLE_CONTROL_MAX_FRAME_BYTES, CHRONICLE_CONTROL_VERSION, + LeaseAcquireOutcome, TrajectoryAppendRequest, TrajectoryAppendResponse, + CHRONICLE_CONTROL_MAX_FRAME_BYTES, CHRONICLE_CONTROL_VERSION, +}; +use persisting_pchronicle::{ + AttemptRecord, AttemptRecordState, AttemptRegistry, RawEventLanceStore, RunControlStore, + StoryCoords, }; -use persisting_pchronicle::{AttemptRegistry, RunControlStore}; use std::io::Write; use std::net::SocketAddr; use std::sync::Arc; @@ -181,7 +186,7 @@ async fn handle_request( Request::GetRun { run_id } => Response::Run(control.get(&run_id).await?), Request::ListRuns => Response::Runs(control.list().await?), Request::GetAttempt { run_id } => { - Response::Attempt(transcode(attempts.get(&run_id).await?)?) + Response::Attempt(attempts.get(&run_id).await?.map(map_attempt_record)) } Request::PublishAttemptActive { run_id, @@ -213,13 +218,11 @@ async fn handle_request( .publish_terminal(&run_id, &attempt_id, lease_epoch, result) .await?, ), - Request::AppendTrajectory(request) => { - let request = transcode(request)?; - let response = persisting_pchronicle::operations::trajectory::append_async(request) + Request::AppendTrajectory(request) => Response::TrajectoryAppend( + append_trajectory(request) .await - .context("append trajectory")?; - Response::TrajectoryAppend(transcode(response)?) - } + .context("append trajectory")?, + ), }) } @@ -256,6 +259,46 @@ fn map_commit_outcome(value: persisting_pchronicle::CommitRunOutcome) -> CommitR } } -fn transcode(value: T) -> Result { - serde_json::from_value(serde_json::to_value(value)?).context("translate control protocol value") +fn map_attempt_record(value: AttemptRecord) -> ProtocolAttemptRecord { + ProtocolAttemptRecord { + revision: value.revision, + run_id: value.run_id, + attempt_id: value.attempt_id, + lease_epoch: value.lease_epoch, + state: match value.state { + AttemptRecordState::Active => ProtocolAttemptRecordState::Active, + AttemptRecordState::Terminal => ProtocolAttemptRecordState::Terminal, + }, + heartbeat_at_unix_ms: value.heartbeat_at_unix_ms, + expires_at_unix_ms: value.expires_at_unix_ms, + terminal_result: value.terminal_result, + } +} + +pub(crate) async fn append_trajectory( + request: TrajectoryAppendRequest, +) -> Result { + let session = StoryCoords::new( + &request.storage, + &request.agent_id, + &request.session_id, + request.root_session_id.clone(), + ); + let store = RawEventLanceStore; + let accepted_records = request.records.len(); + let note = if request.records.is_empty() { + "No non-empty records; storage unchanged.".to_string() + } else { + let outcome = store.append_events(&session, &request.records).await?; + format!("canonical Lance event log. {}", outcome.note) + }; + Ok(TrajectoryAppendResponse { + dataset: store.display_path(&session)?, + storage: request.storage, + agent_id: request.agent_id, + session_id: request.session_id, + accepted_records, + status: "ok".into(), + note, + }) } diff --git a/crates/persisting-pchronicle-cli/src/tests.rs b/crates/persisting-pchronicle-cli/src/tests.rs index d10061ff..57f04903 100644 --- a/crates/persisting-pchronicle-cli/src/tests.rs +++ b/crates/persisting-pchronicle-cli/src/tests.rs @@ -1,4 +1,37 @@ use super::*; + +#[tokio::test] +async fn trajectory_append_uses_persisting_events_protocol_directly() -> Result<()> { + let temporary = tempfile::tempdir()?; + let request = persisting_events::TrajectoryAppendRequest { + storage: temporary.path().to_string_lossy().into_owned(), + agent_id: "agent".into(), + session_id: "session".into(), + root_session_id: None, + records: vec![persisting_events::EventRecord { + identity: persisting_events::EventIdentity::default(), + seq: 1, + source: "test".into(), + kind: "note".into(), + timestamp: None, + session_id: Some("session".into()), + agent_id: Some("agent".into()), + parent_uuid: None, + trace_id: None, + call_id: None, + subagent_id: None, + parent_agent_id: None, + branch: None, + parent_call_id: None, + payload: serde_json::json!({"content": "hello"}), + }], + }; + let response: persisting_events::TrajectoryAppendResponse = + crate::control::append_trajectory(request).await?; + assert_eq!(response.accepted_records, 1); + assert_eq!(response.status, "ok"); + Ok(()) +} use clap::CommandFactory; use serde_json::Value; use std::fs; diff --git a/crates/persisting-pchronicle/src/messages.rs b/crates/persisting-pchronicle/src/messages.rs index 9b336c0a..b58b4d78 100644 --- a/crates/persisting-pchronicle/src/messages.rs +++ b/crates/persisting-pchronicle/src/messages.rs @@ -275,132 +275,6 @@ pub struct SearchImportLanceResponse { pub note: String, } -// --------------------------------------------------------------------------- -// Trajectory -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryAppendRequest { - pub storage: String, - /// Stable agent identity (directory segment under `storage/`). - pub agent_id: String, - /// Session / run scope within the agent (directory segment under `storage/agent_id/`). - pub session_id: String, - /// When set, nested subagent sessions live under `{root_session_id}/subagents/{session_id}/`. - #[serde(default)] - pub root_session_id: Option, - /// Typed canonical records. - pub records: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryAppendResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub accepted_records: usize, - pub dataset: String, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryReplayRequest { - pub storage: String, - pub agent_id: String, - pub session_id: String, - #[serde(default)] - pub offset: usize, - pub limit: Option, - /// When set, read nested session at `{root_session_id}/subagents/{session_id}/`. - #[serde(default)] - pub root_session_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryReplayResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub records: Vec, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryStatsRequest { - pub storage: String, - pub agent_id: String, - pub session_id: String, - /// When set, read nested session at `{root_session_id}/subagents/{session_id}/`. - #[serde(default)] - pub root_session_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryStatsResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub dataset: String, - /// Row count in the event log (`0` when file is missing). - pub row_count: usize, - /// Physical manifest revision observed by the stats read. - pub manifest_revision: Option, - /// Extra physical rows sharing a non-null event_id. This is diagnostic - /// only; canonical events remain at-least-once and are never hidden. - #[serde(default)] - pub duplicate_event_ids: usize, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryMaterializeRequest { - pub storage: String, - pub agent_id: String, - pub session_id: String, - #[serde(default)] - pub root_session_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryMaterializeResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub markdown_path: String, - pub event_rows: usize, - pub markdown_blocks: usize, - pub skipped_events: usize, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryExtractRequest { - pub storage: String, - pub agent_id: String, - pub session_id: String, - #[serde(default)] - pub root_session_id: Option, - pub out_dir: String, - /// When set on a capture run root story, copy the full run including `subagents/`. - #[serde(default)] - pub include_subagents: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryExtractResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub out_dir: String, - pub files_copied: usize, - pub status: String, - pub note: String, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub enum RequestBody { SearchAdd(SearchAddRequest), diff --git a/crates/persisting-pchronicle/src/operations/mod.rs b/crates/persisting-pchronicle/src/operations/mod.rs index e70ebb58..01ca5522 100644 --- a/crates/persisting-pchronicle/src/operations/mod.rs +++ b/crates/persisting-pchronicle/src/operations/mod.rs @@ -1,7 +1,6 @@ -//! Search adapters and typed trajectory operations. +//! Search adapters. #[cfg(feature = "search")] pub mod bridge; #[cfg(feature = "search")] pub mod dispatch; -pub mod trajectory; diff --git a/crates/persisting-pchronicle/src/operations/trajectory/mod.rs b/crates/persisting-pchronicle/src/operations/trajectory/mod.rs deleted file mode 100644 index 80d185fd..00000000 --- a/crates/persisting-pchronicle/src/operations/trajectory/mod.rs +++ /dev/null @@ -1,193 +0,0 @@ -//! Typed trajectory operations over pChronicle services. -//! -//! pChronicle owns **Lance** (canonical event log) and **AgenticMD** (optional -//! human-readable view). This module maps -//! protocol requests to those pChronicle operations. -//! -//! Path: `{storage}/{agent_id}/{run_id}/` with `{session_id}.md` per logical session. -//! -use crate::{ - export_story_bundle, layer_stats, materialize_lance_to_markdown, RawEventLanceStore, - StoryCoords, -}; -pub use crate::{ - TrajectoryAppendRequest, TrajectoryAppendResponse, TrajectoryExtractRequest, - TrajectoryExtractResponse, TrajectoryMaterializeRequest, TrajectoryMaterializeResponse, - TrajectoryReplayRequest, TrajectoryReplayResponse, TrajectoryStatsRequest, - TrajectoryStatsResponse, -}; -use anyhow::Result; - -fn session_from_request( - storage: &str, - agent_id: &str, - session_id: &str, - root_session_id: Option<&str>, -) -> StoryCoords { - StoryCoords::new( - storage, - agent_id, - session_id, - root_session_id.map(str::to_string), - ) -} - -pub async fn materialize_async( - request: TrajectoryMaterializeRequest, -) -> Result { - let root_session_id = request.root_session_id.as_deref(); - let session = session_from_request( - &request.storage, - &request.agent_id, - &request.session_id, - root_session_id, - ); - let outcome = materialize_lance_to_markdown(&session).await?; - Ok(TrajectoryMaterializeResponse { - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - markdown_path: outcome.markdown_path, - event_rows: outcome.stats.source_events, - markdown_blocks: outcome.stats.markdown_blocks, - skipped_events: outcome.stats.skipped_events, - status: "ok".to_string(), - note: outcome.note, - }) -} - -pub async fn append_async(request: TrajectoryAppendRequest) -> Result { - let root_session_id = request.root_session_id.as_deref(); - let session = session_from_request( - &request.storage, - &request.agent_id, - &request.session_id, - root_session_id, - ); - let store = RawEventLanceStore; - let accepted_records = request.records.len(); - let (dataset, note) = if request.records.is_empty() { - ( - store.display_path(&session)?, - "No non-empty records; storage unchanged.".to_string(), - ) - } else { - let outcome = store.append_events(&session, &request.records).await?; - ( - store.display_path(&session)?, - format!("canonical Lance event log. {}", outcome.note), - ) - }; - - Ok(TrajectoryAppendResponse { - dataset, - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - accepted_records, - status: "ok".to_string(), - note, - }) -} - -pub async fn replay_async(request: TrajectoryReplayRequest) -> Result { - let root_session_id = request.root_session_id.as_deref(); - let session = session_from_request( - &request.storage, - &request.agent_id, - &request.session_id, - root_session_id, - ); - - let outcome = RawEventLanceStore - .replay(&session, request.offset, request.limit) - .await?; - - Ok(TrajectoryReplayResponse { - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - records: outcome.records, - status: "ok".to_string(), - note: outcome.note, - }) -} - -pub async fn stats_async(request: TrajectoryStatsRequest) -> Result { - let root_session_id = request.root_session_id.as_deref(); - let session = session_from_request( - &request.storage, - &request.agent_id, - &request.session_id, - root_session_id, - ); - - let layers = layer_stats(&session).await?; - let projection_note = if layers.markdown_blocks > 0 { - format!( - "; AgenticMD debug view {} block(s){}", - layers.markdown_blocks, - layers - .markdown_path - .as_deref() - .map(|path| format!(" at {path}")) - .unwrap_or_default() - ) - } else { - "; no AgenticMD debug view".to_string() - }; - let duplicate_event_ids = duplicate_event_id_count(&session).await?; - Ok(TrajectoryStatsResponse { - dataset: layers.event_log_path, - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - row_count: layers.event_rows, - manifest_revision: RawEventLanceStore.stats(&session).await?.manifest_revision, - duplicate_event_ids, - status: if layers.event_rows > 0 { "ok" } else { "empty" }.into(), - note: format!( - "Canonical Lance event log: {} row(s){projection_note}", - layers.event_rows - ), - }) -} - -async fn duplicate_event_id_count(session: &StoryCoords) -> Result { - let records = crate::RawEventLanceStore - .read_events(session, 0, None) - .await?; - let mut counts = std::collections::HashMap::::new(); - for event_id in records - .into_iter() - .filter_map(|record| record.identity.event_id) - { - *counts.entry(event_id).or_default() += 1; - } - Ok(counts.values().map(|count| count.saturating_sub(1)).sum()) -} - -pub async fn extract_async(request: TrajectoryExtractRequest) -> Result { - let root_session_id = request.root_session_id.as_deref(); - let session = session_from_request( - &request.storage, - &request.agent_id, - &request.session_id, - root_session_id, - ); - let out = std::path::Path::new(&request.out_dir); - let outcome = export_story_bundle(&session, out, request.include_subagents)?; - - Ok(TrajectoryExtractResponse { - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - out_dir: outcome.out_dir, - files_copied: outcome.files_copied, - status: "ok".to_string(), - note: outcome.note, - }) -} - -#[cfg(test)] -mod tests; diff --git a/crates/persisting-pchronicle/src/operations/trajectory/tests.rs b/crates/persisting-pchronicle/src/operations/trajectory/tests.rs deleted file mode 100644 index e8bb51dd..00000000 --- a/crates/persisting-pchronicle/src/operations/trajectory/tests.rs +++ /dev/null @@ -1,876 +0,0 @@ -use super::*; -use crate::{ - expand_story_locations, layer_stats, resolve_story_read_location, - session_markdown_write_path_for_key, story_lance_event_path as trajectory_event_log_path, - story_run_dir as trajectory_run_dir, EventRecord, -}; - -struct Call { - call_id: String, - trace_id: String, - started_at: String, -} - -fn event_record( - kind: &str, - session_id: Option, - agent_id: Option, - payload: serde_json::Value, -) -> EventRecord { - EventRecord { - identity: Default::default(), - seq: 0, - source: "pchronicle-test".into(), - kind: kind.into(), - timestamp: Some("2026-01-01T00:00:00Z".into()), - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload, - } -} - -fn llm_request_record( - session_id: Option, - agent_id: Option, - model: &str, - path: &str, - body: &serde_json::Value, -) -> EventRecord { - event_record( - "llm.request", - session_id, - agent_id, - serde_json::json!({"model": model, "path": path, "body": body}), - ) -} - -fn llm_response_record( - session_id: Option, - agent_id: Option, - status: u16, - body: &serde_json::Value, - streaming: bool, - call: &Call, -) -> EventRecord { - let mut record = event_record( - if streaming { - "llm.response.stream" - } else { - "llm.response" - }, - session_id, - agent_id, - serde_json::json!({"status": status, "body": body}), - ); - record.call_id = Some(call.call_id.clone()); - record.trace_id = Some(call.trace_id.clone()); - record.timestamp = Some(call.started_at.clone()); - record -} - -fn record_to_event_line(record: &EventRecord) -> anyhow::Result { - serde_json::to_string(record).map_err(anyhow::Error::from) -} - -#[test] -fn rejects_bad_segments() { - assert!(trajectory_event_log_path("/tmp", "a/b", "s", None).is_err()); - assert!(trajectory_event_log_path("/tmp", "..", "s", None).is_err()); - let nested = trajectory_event_log_path("/tmp", "agent", "sub-1", Some("root-1")).unwrap(); - assert!(nested.ends_with("agent/root-1/events.lance")); - let root = trajectory_event_log_path("/tmp", "agent", "root-1", Some("root-1")).unwrap(); - assert!(root.ends_with("agent/root-1/events.lance")); -} - -#[tokio::test] -async fn append_replay_stats_lance_roundtrip() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("traj_store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - - let line1 = record_to_event_line(&EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"step 1"}), - }) - .unwrap(); - let line2 = record_to_event_line(&EventRecord { - identity: Default::default(), - seq: 1, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"step 2"}), - }) - .unwrap(); - - let append = append_async(TrajectoryAppendRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sess_1".into(), - root_session_id: None, - records: decode_test_records(&format!("{line1}\n{line2}\n")), - }) - .await - .unwrap(); - assert_eq!(append.accepted_records, 2); - assert!(append.note.contains("Lance:")); - let lance_path = trajectory_event_log_path(&storage_s, "agent_a", "sess_1", None).unwrap(); - assert!(lance_path.is_dir(), "expected {}", lance_path.display()); - - let replay = replay_async(TrajectoryReplayRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sess_1".into(), - offset: 0, - limit: Some(10), - root_session_id: None, - }) - .await - .unwrap(); - assert_eq!(replay.records.len(), 2); - - let st = stats_async(TrajectoryStatsRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sess_1".into(), - root_session_id: None, - }) - .await - .unwrap(); - assert_eq!(st.row_count, 2); - assert!(st.note.contains("Canonical Lance event log: 2")); -} - -#[tokio::test] -async fn append_replay_stats_nested_lance_roundtrip() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("traj_store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - - let mk = |content: &str| { - record_to_event_line(&EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ "content": content }), - }) - .unwrap() - }; - - let append = append_async(TrajectoryAppendRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sub-1".into(), - root_session_id: Some("root-1".into()), - records: decode_test_records(&format!("{}\n{}\n{}\n", mk("a"), mk("b"), mk("c"))), - }) - .await - .unwrap(); - assert_eq!(append.accepted_records, 3); - - append_async(TrajectoryAppendRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sub-2".into(), - root_session_id: Some("root-1".into()), - records: decode_test_records(&format!("{}\n{}\n", mk("x"), mk("y"))), - }) - .await - .unwrap(); - - let replay = replay_async(TrajectoryReplayRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sub-1".into(), - offset: 1, - limit: Some(1), - root_session_id: Some("root-1".into()), - }) - .await - .unwrap(); - assert_eq!(replay.records.len(), 1); - - let st = stats_async(TrajectoryStatsRequest { - storage: storage_s.clone(), - agent_id: "agent_a".into(), - session_id: "sub-1".into(), - root_session_id: Some("root-1".into()), - }) - .await - .unwrap(); - assert_eq!(st.row_count, 3); - assert!(st.dataset.contains("events.lance")); - - let other = replay_async(TrajectoryReplayRequest { - storage: storage_s, - agent_id: "agent_a".into(), - session_id: "sub-2".into(), - offset: 0, - limit: None, - root_session_id: Some("root-1".into()), - }) - .await - .unwrap(); - assert_eq!(other.records.len(), 2); -} - -#[tokio::test] -async fn materialized_markdown_does_not_replace_canonical_replay() { - let call = Call { - call_id: "c".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("traj_store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - - let req = llm_request_record( - Some("s".into()), - None, - "m", - "/v1/chat", - &serde_json::json!({"messages":[{"role":"user","content":"first"}]}), - ); - let resp = llm_response_record( - Some("s".into()), - None, - 200, - &serde_json::json!({"choices":[{"message":{"role":"assistant","content":"second"}}]}), - false, - &call, - ); - let records = decode_test_records(&format!( - "{}\n{}\n", - record_to_event_line(&req).unwrap(), - record_to_event_line(&resp).unwrap() - )); - - append_async(TrajectoryAppendRequest { - storage: storage_s.clone(), - agent_id: "a".into(), - session_id: "s".into(), - root_session_id: None, - records: records.clone(), - }) - .await - .unwrap(); - - let session = StoryCoords::new(storage_s.clone(), "a", "s", None); - materialize_lance_to_markdown(&session).await.unwrap(); - - let replay = replay_async(TrajectoryReplayRequest { - storage: storage_s.clone(), - agent_id: "a".into(), - session_id: "s".into(), - offset: 1, - limit: Some(1), - root_session_id: None, - }) - .await - .unwrap(); - assert_eq!(replay.records.len(), 1); - let row = serde_json::to_value(&replay.records[0]).unwrap(); - assert_eq!(row["kind"], "llm.response"); - - let md_path = session_markdown_write_path_for_key( - &trajectory_run_dir(&storage_s, "a", "s", None).unwrap(), - "s", - ); - let md_text = std::fs::read_to_string(&md_path).unwrap(); - assert!(md_text.contains("\n\nmessage body\n\n"; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AgenticmdHeader { +pub struct MarkdownHeader { #[serde(rename = "type", default = "default_block_type")] pub type_name: String, #[serde(default)] @@ -40,12 +40,12 @@ pub struct AgenticmdHeader { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AgenticmdBlock { - pub header: AgenticmdHeader, +pub struct MarkdownBlock { + pub header: MarkdownHeader, pub body: String, } -impl AgenticmdBlock { +impl MarkdownBlock { /// Legacy presentation role, derived from Storyline `source` when absent. pub fn role(&self) -> Option<&str> { if let Some(role) = self.header.fields.get("role").and_then(|v| v.as_str()) { @@ -80,7 +80,7 @@ impl AgenticmdBlock { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AgenticmdDocument { +pub struct MarkdownDocument { /// Logical pChronicle format name (`agenticmd`). pub format: String, /// Frontmatter `format:` value (usually `persisting`). @@ -91,11 +91,11 @@ pub struct AgenticmdDocument { pub agent_id: Option, #[serde(default)] pub frontmatter: BTreeMap, - pub blocks: Vec, + pub blocks: Vec, } -impl AgenticmdDocument { - pub fn new(blocks: Vec) -> Self { +impl MarkdownDocument { + pub fn new(blocks: Vec) -> Self { Self { format: AGENTICMD_FORMAT_NAME.into(), frontmatter_format: AGENTICMD_FRONTMATTER_FORMAT.into(), @@ -107,10 +107,10 @@ impl AgenticmdDocument { } } -pub fn parse_agenticmd_document(input: &str) -> Result { +pub fn parse_agenticmd_document(input: &str) -> Result { let (frontmatter, _body, _off) = split_frontmatter_with_offset(input)?; let spans = parse_agenticmd_blocks_with_spans(input)?; - let mut doc = AgenticmdDocument::new(spans.into_iter().map(|s| s.block).collect()); + let mut doc = MarkdownDocument::new(spans.into_iter().map(|s| s.block).collect()); doc.frontmatter = frontmatter; if let Some(fmt) = doc.frontmatter.get("format").and_then(Value::as_str) { doc.frontmatter_format = fmt.to_string(); @@ -142,14 +142,14 @@ pub fn parse_agenticmd_document(input: &str) -> Result { /// `start..end` covers the comment line through trailing blank lines — the same /// span capture uses for markdown upsert rewrites. #[derive(Debug, Clone, PartialEq)] -pub struct AgenticmdBlockSpan { - pub block: AgenticmdBlock, +pub struct MarkdownBlockSpan { + pub block: MarkdownBlock, pub start: usize, pub end: usize, } /// Parse blocks with absolute byte spans (for capture upsert / diagnostics). -pub fn parse_agenticmd_blocks_with_spans(input: &str) -> Result> { +pub fn parse_agenticmd_blocks_with_spans(input: &str) -> Result> { let (_frontmatter, body, body_offset) = split_frontmatter_with_offset(input)?; parse_blocks_with_spans(body, body_offset) } @@ -176,8 +176,8 @@ pub fn encode_agenticmd_preamble(frontmatter: &T) -> Result Result { - let header = AgenticmdHeader { +pub fn encode_agenticmd_block(block: &MarkdownBlock) -> Result { + let header = MarkdownHeader { type_name: block.header.type_name.clone(), length: block.body.len(), fields: block.header.fields.clone(), @@ -235,16 +235,16 @@ mod strict_frontmatter_tests { } } -fn parse_blocks_with_spans(input: &str, base_offset: usize) -> Result> { +fn parse_blocks_with_spans(input: &str, base_offset: usize) -> Result> { if input.trim().is_empty() { return Ok(Vec::new()); } if !input.contains(BLOCK_MARKER) { let body = input.trim().to_string(); let fields = BTreeMap::from([("source".into(), Value::String("system".into()))]); - return Ok(vec![AgenticmdBlockSpan { - block: AgenticmdBlock { - header: AgenticmdHeader { + return Ok(vec![MarkdownBlockSpan { + block: MarkdownBlock { + header: MarkdownHeader { type_name: default_block_type(), length: body.len(), fields, @@ -307,8 +307,8 @@ fn parse_blocks_with_spans(input: &str, base_offset: usize) -> Result Result Result<(AgenticmdHeader, Option)> { +fn parse_block_comment(line: &str) -> Result<(MarkdownHeader, Option)> { let after = line .strip_prefix(BLOCK_MARKER) .ok_or_else(|| Error::Other("missing persisting:block marker".into()))?; @@ -335,7 +335,7 @@ fn parse_block_comment(line: &str) -> Result<(AgenticmdHeader, Option)> { ); } return Ok(( - AgenticmdHeader { + MarkdownHeader { type_name: default_block_type(), length: 0, fields, @@ -357,7 +357,7 @@ fn parse_block_comment(line: &str) -> Result<(AgenticmdHeader, Option)> { .get("length") .and_then(Value::as_u64) .map(|n| n as usize); - let mut header: AgenticmdHeader = serde_json::from_value(raw)?; + let mut header: MarkdownHeader = serde_json::from_value(raw)?; if !speaker.is_empty() && !header.fields.contains_key("source") && !header.fields.contains_key("role") diff --git a/crates/persisting-pchronicle/src/agenticmd/convert.rs b/crates/persisting-pchronicle/src/agenticmd/convert.rs index bf987cbe..08bff45b 100644 --- a/crates/persisting-pchronicle/src/agenticmd/convert.rs +++ b/crates/persisting-pchronicle/src/agenticmd/convert.rs @@ -11,8 +11,8 @@ use crate::formats::storyline::{StorylineAgent, StorylineDocument, StorylineTurn use crate::{DocumentFormat, Error, Result}; use super::codec::{ - encode_agenticmd_block, encode_agenticmd_preamble, parse_agenticmd_document, AgenticmdBlock, - AgenticmdDocument, AgenticmdHeader, AGENTICMD_FRONTMATTER_FORMAT, + encode_agenticmd_block, encode_agenticmd_preamble, parse_agenticmd_document, MarkdownBlock, + MarkdownDocument, MarkdownHeader, AGENTICMD_FRONTMATTER_FORMAT, }; const STORYLINE_METADATA_KEY: &str = "storyline"; @@ -120,7 +120,7 @@ pub(super) fn encode_storyline_preamble(story: &StorylineDocument) -> Result, -) -> Result { +) -> Result { let mut turn_metadata = serde_json::to_value(turn)? .as_object() .cloned() @@ -139,8 +139,8 @@ pub(super) fn storyline_turn_block( if let Some(edit_key) = edit_key { fields.insert("call_id".into(), Value::String(edit_key.into())); } - Ok(AgenticmdBlock { - header: AgenticmdHeader { + Ok(MarkdownBlock { + header: MarkdownHeader { type_name: type_name.into(), length: body.len(), fields, @@ -149,7 +149,7 @@ pub(super) fn storyline_turn_block( }) } -fn agenticmd_to_storyline(doc: &AgenticmdDocument) -> Result { +fn agenticmd_to_storyline(doc: &MarkdownDocument) -> Result { let session_id = doc.session_id.clone().unwrap_or_else(|| "unknown".into()); let agent_id = doc.agent_id.clone().unwrap_or_else(|| "unknown".into()); diff --git a/crates/persisting-pchronicle/src/agenticmd/fs.rs b/crates/persisting-pchronicle/src/agenticmd/fs.rs index 6bf3eef4..5a43d53b 100644 --- a/crates/persisting-pchronicle/src/agenticmd/fs.rs +++ b/crates/persisting-pchronicle/src/agenticmd/fs.rs @@ -10,7 +10,7 @@ use serde::Serialize; use super::codec::{ encode_agenticmd_block, encode_agenticmd_preamble, parse_agenticmd_blocks_with_spans, - parse_agenticmd_document, AgenticmdBlock, AgenticmdBlockSpan, AgenticmdHeader, + parse_agenticmd_document, MarkdownBlock, MarkdownBlockSpan, MarkdownHeader, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, }; use super::convert::{encode_storyline_preamble, storyline_turn_block}; @@ -42,7 +42,7 @@ fn default_document_preamble() -> Result { } /// Encode one generated block after minimal comment-safety validation. -pub fn encode_agenticmd_block_validated(block: &AgenticmdBlock) -> Result { +pub fn encode_agenticmd_block_validated(block: &MarkdownBlock) -> Result { validate_type_name(&block.header.type_name)?; validate_speaker(block_speaker(&block.header))?; let mut block = block.clone(); @@ -51,7 +51,7 @@ pub fn encode_agenticmd_block_validated(block: &AgenticmdBlock) -> Result Result> { +pub fn parse_agenticmd_document_validated(input: &str) -> Result> { parse_agenticmd_document(input) .map_err(|e| anyhow::anyhow!("agenticmd parse: {e}"))? .blocks @@ -65,14 +65,14 @@ pub fn parse_agenticmd_document_validated(input: &str) -> Result Result> { +pub fn parse_agenticmd_spans_validated(input: &str) -> Result> { let spans = parse_agenticmd_blocks_with_spans(input) .map_err(|e| anyhow::anyhow!("agenticmd span parse: {e}"))?; spans .into_iter() .enumerate() .map(|(i, span)| { - let AgenticmdBlockSpan { block, start, end } = span; + let MarkdownBlockSpan { block, start, end } = span; validate_agenticmd_block(&block) .with_context(|| format!("agenticmd span block[{i}]"))?; Ok((block, start, end)) @@ -86,7 +86,7 @@ pub fn parse_agenticmd_spans_validated(input: &str) -> Result, ) -> Result { if blocks.is_empty() { @@ -118,7 +118,7 @@ pub fn append_agenticmd_blocks( pub fn write_agenticmd_document( path: &Path, preamble: &str, - blocks: &[AgenticmdBlock], + blocks: &[MarkdownBlock], ) -> Result<()> { let mut output = preamble.to_string(); for block in blocks { @@ -257,7 +257,7 @@ pub fn count_agenticmd_role(path: &Path, role: &str) -> Result { /// Replace the block whose header `call_id` and presentation role match, or append when missing. /// /// Returns `true` when an existing block was rewritten. -pub fn upsert_block_by_call_id(path: &Path, call_id: &str, block: AgenticmdBlock) -> Result { +pub fn upsert_block_by_call_id(path: &Path, call_id: &str, block: MarkdownBlock) -> Result { if call_id.trim().is_empty() { bail!("call_id must not be empty for markdown upsert"); } @@ -284,7 +284,7 @@ pub fn find_block_by_call_id_and_role( bytes: &[u8], call_id: &str, role: &str, -) -> Result> { +) -> Result> { let text = std::str::from_utf8(bytes).context("markdown upsert requires UTF-8 document")?; for (block, start, end) in parse_agenticmd_spans_validated(text)? { if block_matches_upsert_key(&block.header, call_id, role) { @@ -294,12 +294,12 @@ pub fn find_block_by_call_id_and_role( Ok(None) } -fn block_matches_upsert_key(header: &AgenticmdHeader, call_id: &str, role: &str) -> bool { +fn block_matches_upsert_key(header: &MarkdownHeader, call_id: &str, role: &str) -> bool { header.fields.get("call_id").and_then(|v| v.as_str()) == Some(call_id) && header_role(header) == role } -fn header_role(header: &AgenticmdHeader) -> &str { +fn header_role(header: &MarkdownHeader) -> &str { if let Some(role) = header.fields.get("role").and_then(|v| v.as_str()) { return role; } @@ -328,7 +328,7 @@ pub fn rewrite_block_range(path: &Path, start: usize, end: usize, new_block: &[u } /// Strict-parse all agenticmd blocks from a markdown file (empty if missing). -pub fn read_agenticmd_blocks_from_file(path: &Path) -> Result> { +pub fn read_agenticmd_blocks_from_file(path: &Path) -> Result> { if !path.exists() { return Ok(Vec::new()); } @@ -369,7 +369,7 @@ fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> { #[cfg(test)] mod tests { use super::super::codec::{ - encode_agenticmd_preamble, AgenticmdHeader, AGENTICMD_BLOCK_LAYOUT, + encode_agenticmd_preamble, MarkdownHeader, AGENTICMD_BLOCK_LAYOUT, AGENTICMD_FRONTMATTER_FORMAT, }; use super::*; @@ -392,13 +392,13 @@ mod tests { .unwrap() } - fn block_with_call(call_id: &str, role: &str, body: &str) -> AgenticmdBlock { + fn block_with_call(call_id: &str, role: &str, body: &str) -> MarkdownBlock { let mut fields = BTreeMap::new(); fields.insert("role".into(), json!(role)); fields.insert("kind".into(), json!("llm.response.stream")); fields.insert("call_id".into(), json!(call_id)); - AgenticmdBlock { - header: AgenticmdHeader { + MarkdownBlock { + header: MarkdownHeader { type_name: "markdown".into(), length: body.len(), fields, @@ -407,27 +407,27 @@ mod tests { } } - fn block_header(role: &str, kind: &str) -> AgenticmdHeader { + fn block_header(role: &str, kind: &str) -> MarkdownHeader { let mut fields = BTreeMap::new(); fields.insert("role".into(), json!(role)); fields.insert("kind".into(), json!(kind)); fields.insert("session_id".into(), json!("test-session")); - AgenticmdHeader { + MarkdownHeader { type_name: "markdown".into(), length: 0, fields, } } - fn encode_block(header: AgenticmdHeader, body: &str) -> String { - encode_agenticmd_block_validated(&AgenticmdBlock { + fn encode_block(header: MarkdownHeader, body: &str) -> String { + encode_agenticmd_block_validated(&MarkdownBlock { header, body: body.into(), }) .unwrap() } - fn canonical_doc(blocks: &[(AgenticmdHeader, &str)]) -> String { + fn canonical_doc(blocks: &[(MarkdownHeader, &str)]) -> String { let mut doc = baseline_preamble(); for (header, body) in blocks { doc.push_str(&encode_block(header.clone(), body)); @@ -435,7 +435,7 @@ mod tests { doc } - fn read_blocks(path: &Path) -> Vec { + fn read_blocks(path: &Path) -> Vec { let text = std::fs::read_to_string(path).unwrap(); parse_agenticmd_document_validated(&text).unwrap() } @@ -595,8 +595,8 @@ mod tests { fields.insert("kind".into(), json!("llm.response")); fields.insert("call_id".into(), json!("c1")); let body = "hello world"; - let via_validated = encode_agenticmd_block_validated(&AgenticmdBlock { - header: AgenticmdHeader { + let via_validated = encode_agenticmd_block_validated(&MarkdownBlock { + header: MarkdownHeader { type_name: "text".into(), length: 0, fields: fields.clone(), @@ -604,8 +604,8 @@ mod tests { body: body.into(), }) .unwrap(); - let via_raw = super::super::codec::encode_agenticmd_block(&AgenticmdBlock { - header: AgenticmdHeader { + let via_raw = super::super::codec::encode_agenticmd_block(&MarkdownBlock { + header: MarkdownHeader { type_name: "text".into(), length: body.len(), fields, diff --git a/crates/persisting-pchronicle/src/agenticmd/mod.rs b/crates/persisting-pchronicle/src/agenticmd/mod.rs index b3464e66..a7bc899e 100644 --- a/crates/persisting-pchronicle/src/agenticmd/mod.rs +++ b/crates/persisting-pchronicle/src/agenticmd/mod.rs @@ -21,7 +21,6 @@ pub use layout::{ }; #[cfg(feature = "lance-store")] pub use projection::{ - event_records_to_storyline, layer_stats, materialize_lance_to_markdown, - materialize_markdown_path, write_markdown_projection, LayerStats, MaterializeOutcome, - MaterializeStats, + layer_stats, materialize_lance_to_markdown, materialize_markdown_path, + write_markdown_projection, LayerStats, MaterializeOutcome, MaterializeStats, }; diff --git a/crates/persisting-pchronicle/src/agenticmd/projection.rs b/crates/persisting-pchronicle/src/agenticmd/projection.rs index b5c31db5..8f41f2a2 100644 --- a/crates/persisting-pchronicle/src/agenticmd/projection.rs +++ b/crates/persisting-pchronicle/src/agenticmd/projection.rs @@ -5,9 +5,10 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; -use crate::{ - project_event_records, EventRecord, RawEventLanceStore, StoryCoords, StorylineDocument, -}; +use crate::convert::project_event_records; +use crate::formats::{EventRecord, StorylineDocument}; +use crate::layout::StoryCoords; +use crate::store::RawEventLanceStore; use super::fs::{agenticmd_block_count, write_agenticmd_storyline}; use super::layout::{locate_session_markdown_for_key, session_markdown_path_for_key}; diff --git a/crates/persisting-pchronicle/src/agenticmd/validate.rs b/crates/persisting-pchronicle/src/agenticmd/validate.rs index 7a684afd..10614a5b 100644 --- a/crates/persisting-pchronicle/src/agenticmd/validate.rs +++ b/crates/persisting-pchronicle/src/agenticmd/validate.rs @@ -5,9 +5,9 @@ use anyhow::{bail, Result}; -use super::codec::{AgenticmdBlock, AgenticmdHeader}; +use super::codec::{MarkdownBlock, MarkdownHeader}; -pub fn block_speaker(header: &AgenticmdHeader) -> &str { +pub fn block_speaker(header: &MarkdownHeader) -> &str { header .fields .get("source") @@ -47,7 +47,7 @@ pub fn validate_type_name(type_name: &str) -> Result<()> { Ok(()) } -pub fn validate_agenticmd_block(block: &AgenticmdBlock) -> Result<()> { +pub fn validate_agenticmd_block(block: &MarkdownBlock) -> Result<()> { validate_type_name(&block.header.type_name)?; validate_speaker(block_speaker(&block.header))?; Ok(()) diff --git a/crates/persisting-pchronicle/src/append_queue.rs b/crates/persisting-pchronicle/src/append_queue.rs index 1ccfe889..1bdf1100 100644 --- a/crates/persisting-pchronicle/src/append_queue.rs +++ b/crates/persisting-pchronicle/src/append_queue.rs @@ -7,8 +7,10 @@ use std::time::Duration; use anyhow::Context; use thiserror::Error; +use crate::formats::EventRecord; +use crate::layout::StoryCoords; use crate::store::compact_sealed_event_segment; -use crate::{raw_event_lance_path, EventRecord, RawEventLanceAppender, StoryCoords}; +use crate::store::{raw_event_lance_path, RawEventLanceAppender}; pub const DEFAULT_RAW_EVENT_QUEUE_CAPACITY: usize = 256; pub const DEFAULT_RAW_EVENT_BATCH_SIZE: usize = 256; @@ -421,7 +423,7 @@ mod tests { use super::*; use serde_json::Value; - use crate::RawEventLanceStore; + use crate::store::RawEventLanceStore; fn event() -> EventRecord { EventRecord { diff --git a/crates/persisting-pchronicle/src/convert/actf.rs b/crates/persisting-pchronicle/src/convert/actf.rs index b5a5dc0b..9109da47 100644 --- a/crates/persisting-pchronicle/src/convert/actf.rs +++ b/crates/persisting-pchronicle/src/convert/actf.rs @@ -169,10 +169,6 @@ pub fn storylines_to_actf(stories: &[StorylineDocument]) -> Result Ok(document) } -pub fn is_actf_storyline(story: &StorylineDocument) -> bool { - residual(story).is_some() -} - fn attempt_to_storyline( document: &ActfDocument, attempt_id: &str, @@ -650,9 +646,9 @@ fn residual(story: &StorylineDocument) -> Option<&Map> { #[cfg(test)] mod tests { use super::*; - use crate::parse_actf_document; + use crate::formats::parse_actf_document; #[cfg(feature = "lance-store")] - use crate::StorylineLanceStore; + use crate::store::StorylineLanceStore; const FIXTURE: &str = r#"{ "task_id":"task-1","category":"software-engineering","k":1, diff --git a/crates/persisting-pchronicle/src/convert/atif.rs b/crates/persisting-pchronicle/src/convert/atif.rs index 6c2b5d6e..a9197b6a 100644 --- a/crates/persisting-pchronicle/src/convert/atif.rs +++ b/crates/persisting-pchronicle/src/convert/atif.rs @@ -225,7 +225,8 @@ pub fn storyline_to_atif(story: &StorylineDocument) -> Result { #[cfg(test)] mod tests { use super::{atif_to_storyline, storyline_to_atif}; - use crate::{AtifTrajectory, DocumentFormat, Error, FieldPresence}; + use crate::atif::AtifTrajectory; + use crate::{DocumentFormat, Error, FieldPresence}; #[test] fn malformed_atif_observation_is_not_silently_dropped() { diff --git a/crates/persisting-pchronicle/src/convert/mod.rs b/crates/persisting-pchronicle/src/convert/mod.rs index eac4d3a9..19972af9 100644 --- a/crates/persisting-pchronicle/src/convert/mod.rs +++ b/crates/persisting-pchronicle/src/convert/mod.rs @@ -8,89 +8,21 @@ //! actf ──────────────────────────────────┘ //! ``` //! -//! [`ChronicleFormat::Events`] has **no string wire form**: `into_storyline` / -//! `from_storyline` / `convert` return an error. Use [`events_to_storyline`] / -//! [`storyline_to_events`] after loading Lance rows into [`EventsDocument`]. +//! Canonical Event and Storyline Lance are accessed through typed storage APIs; +//! string parsing and encoding stay on the four peripheral document formats. mod actf; mod atif; mod events; mod openai_msg; -pub use crate::agenticmd::{encode_agenticmd, parse_agenticmd}; -pub use actf::{ - actf_to_storyline, actf_to_storylines, is_actf_storyline, storyline_to_actf, storylines_to_actf, -}; +pub use actf::{actf_to_storyline, actf_to_storylines, storyline_to_actf, storylines_to_actf}; pub use atif::{atif_to_storyline, storyline_to_atif}; #[cfg(feature = "lance-store")] pub(crate) use events::event_storyline_key; pub use events::{events_to_storyline, project_event_records, storyline_to_events}; pub use openai_msg::{openai_msg_to_storyline, storyline_to_openai_msg}; -use crate::atif::AtifTrajectory; -use crate::format::ChronicleFormat; -use crate::formats::actf::ActfDocument; -use crate::formats::events::events_lance_only_error; -use crate::formats::storyline::StorylineDocument; -use crate::formats::{parse_openai_msg_document, parse_storyline_document}; -use crate::Result; - -/// Parse a supported **string** document into the storyline hub. -/// -/// [`ChronicleFormat::Events`] always errors — load Lance separately, then call -/// [`events_to_storyline`]. -pub fn into_storyline(format: ChronicleFormat, input: &str) -> Result { - match format { - ChronicleFormat::Storyline => parse_storyline_document(input), - ChronicleFormat::Atif => { - let traj = AtifTrajectory::from_json_str(input)?; - atif_to_storyline(&traj) - } - ChronicleFormat::Actf => { - let document = ActfDocument::from_json_str(input)?; - actf_to_storyline(&document) - } - ChronicleFormat::Events => Err(events_lance_only_error()), - ChronicleFormat::Agenticmd => parse_agenticmd(input), - ChronicleFormat::OpenaiMsg => { - let doc = parse_openai_msg_document(input)?; - openai_msg_to_storyline(&doc) - } - } -} - -/// Emit a peripheral/hub format as a string. -/// -/// [`ChronicleFormat::Events`] always errors — use [`storyline_to_events`] then -/// write Lance via Capture, or [`export_events_jsonl`](crate::export_events_jsonl) for debug dumps. -pub fn from_storyline(format: ChronicleFormat, story: &StorylineDocument) -> Result { - match format { - ChronicleFormat::Storyline => story.to_json_string_pretty(), - ChronicleFormat::Atif => Ok(serde_json::to_string_pretty(&storyline_to_atif(story)?)?), - ChronicleFormat::Actf => storyline_to_actf(story)?.to_json_string_pretty(), - ChronicleFormat::Events => Err(events_lance_only_error()), - ChronicleFormat::Agenticmd => encode_agenticmd(story), - ChronicleFormat::OpenaiMsg => { - let doc = storyline_to_openai_msg(story)?; - Ok(serde_json::to_string_pretty(&doc)?) - } - } -} - -/// Convert between two **string** formats via the storyline hub. -/// -/// Any leg involving [`ChronicleFormat::Events`] fails (Lance-only). -pub fn convert(from: ChronicleFormat, to: ChronicleFormat, input: &str) -> Result { - if from == to { - if from.is_lance_only() { - return Err(events_lance_only_error()); - } - return Ok(input.to_string()); - } - let story = into_storyline(from, input)?; - from_storyline(to, &story) -} - pub(crate) fn message_text(message: &serde_json::Value) -> Option { match message { serde_json::Value::String(s) => Some(s.clone()), diff --git a/crates/persisting-pchronicle/src/discovery.rs b/crates/persisting-pchronicle/src/discovery.rs index 50a608fd..a4339d5f 100644 --- a/crates/persisting-pchronicle/src/discovery.rs +++ b/crates/persisting-pchronicle/src/discovery.rs @@ -2,7 +2,8 @@ use anyhow::Result; -use crate::{distinct_session_ids_in_run, StoryCoords}; +use crate::layout::StoryCoords; +use crate::store::distinct_session_ids_in_run; fn is_shared_lance_run_bucket(location: &StoryCoords) -> bool { location diff --git a/crates/persisting-pchronicle/src/document.rs b/crates/persisting-pchronicle/src/document.rs index 1e6507d5..8e4fbbcd 100644 --- a/crates/persisting-pchronicle/src/document.rs +++ b/crates/persisting-pchronicle/src/document.rs @@ -1,13 +1,37 @@ //! Unified read/query entrypoint for pChronicle's physical document formats. +#[cfg(feature = "lance-store")] use std::path::Path; +#[cfg(feature = "lance-store")] use datafusion::prelude::SessionContext; -use crate::{DocumentFormat, Result, StorylineDocument}; +pub use crate::agenticmd::{ + agenticmd_block_count, agenticmd_structural_issues, count_agenticmd_role, encode_agenticmd, + index_agenticmd_path, list_agenticmd_paths, parse_agenticmd, + rewrite_agenticmd_storyline_metadata, upsert_agenticmd_turn, write_agenticmd_storyline, + AgenticmdFileIndex, +}; +pub use crate::convert::{ + actf_to_storyline, actf_to_storylines, atif_to_storyline, events_to_storyline, + openai_msg_to_storyline, project_event_records, storyline_to_actf, storyline_to_atif, + storyline_to_events, storyline_to_openai_msg, storylines_to_actf, +}; +pub use crate::error::{classify_error, Error, ErrorCode, Result}; +pub use crate::format::DocumentFormat; +pub use crate::formats::{ + detect_format, events_lance_only_message, export_events_json_pretty, export_events_jsonl, + is_lossless_openai_storyline, parse_actf_document, parse_openai_msg_corpus_value, + parse_openai_msg_document, parse_storyline_document, recover_openai_msg_files, +}; +pub use crate::interop::{events_to_har, events_to_otlp_json, otlp_json_to_events}; + +#[cfg(feature = "lance-store")] +use crate::formats::StorylineDocument; /// Static filter pushdown guarantee exposed by a document source. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg(feature = "lance-store")] pub enum FilterPushdown { Unsupported, Inexact, @@ -18,6 +42,7 @@ pub enum FilterPushdown { /// Logical tables registered by a source. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg(feature = "lance-store")] pub enum QueryTables { Events, Storyline, @@ -25,6 +50,7 @@ pub enum QueryTables { /// Truthful optimization capabilities for one opened provider. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg(feature = "lance-store")] pub struct QueryCapabilities { pub projection_pushdown: bool, pub filter_pushdown: FilterPushdown, @@ -36,23 +62,28 @@ pub struct QueryCapabilities { } /// Maximum Storyline rows retained by the convenience materialization API. +#[cfg(feature = "lance-store")] pub const DEFAULT_DOCUMENT_MATERIALIZE_ROWS: usize = 10_000; /// Maximum serialized Storyline bytes retained by the convenience materialization API. +#[cfg(feature = "lance-store")] pub const DEFAULT_DOCUMENT_MATERIALIZE_BYTES: usize = 64 * 1024 * 1024; /// One opened physical document source. Provider variants remain private. #[derive(Debug)] +#[cfg(feature = "lance-store")] pub struct DocumentSource { pub(crate) inner: crate::store::DocumentSourceImpl, } /// Open one of the six physical pChronicle document formats. +#[cfg(feature = "lance-store")] pub async fn open_document(format: DocumentFormat, path: &Path) -> Result { Ok(DocumentSource { inner: crate::store::open_document_source(format, path).await?, }) } +#[cfg(feature = "lance-store")] impl DocumentSource { pub fn format(&self) -> DocumentFormat { self.inner.format() diff --git a/crates/persisting-pchronicle/src/format.rs b/crates/persisting-pchronicle/src/format.rs index 99c7e6b0..9bfdcc67 100644 --- a/crates/persisting-pchronicle/src/format.rs +++ b/crates/persisting-pchronicle/src/format.rs @@ -1,10 +1,4 @@ -//! Named storage formats supported by pChronicle. -//! -//! [`ChronicleFormat::Storyline`] is the **hub** interchange format. -//! Peripheral formats convert only to/from storyline — never pairwise. -//! -//! [`ChronicleFormat::Events`] is **Lance-only** (`events.lance`); it is not a -//! JSON/JSONL string format for convert APIs. +//! Named physical document formats supported by pChronicle. use crate::{Error, Result}; use std::fmt; @@ -76,100 +70,6 @@ impl FromStr for DocumentFormat { } } -/// First-class trajectory storage formats. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ChronicleFormat { - /// ATIF-enhanced Run→Storyline→Turn hub (`storyline`). - Storyline, - /// Capture canonical event log — **Lance dataset only** (`events`). - Events, - /// Capture TLV markdown dialogue view (`agenticmd`). - Agenticmd, - /// dlcapt OpenAI-messages step table (`openai_msg`). - OpenaiMsg, - /// Harbor ATIF JSON interchange (`atif`). - Atif, - /// ACTF v1.0 benchmark task/attempt trajectory document (`actf`). - Actf, -} - -impl ChronicleFormat { - pub const ALL: &[ChronicleFormat] = &[ - Self::Storyline, - Self::Events, - Self::Agenticmd, - Self::OpenaiMsg, - Self::Atif, - Self::Actf, - ]; - - pub fn as_str(self) -> &'static str { - match self { - Self::Storyline => "storyline", - Self::Events => "events", - Self::Agenticmd => "agenticmd", - Self::OpenaiMsg => "openai_msg", - Self::Atif => "atif", - Self::Actf => "actf", - } - } - - pub fn is_hub(self) -> bool { - matches!(self, Self::Storyline) - } - - /// `events` has no string wire form (Lance-only). - pub fn is_lance_only(self) -> bool { - matches!(self, Self::Events) - } - - pub fn origin(self) -> &'static str { - match self { - Self::Storyline => "pchronicle (hub)", - Self::Events => "persisting-gateway (Lance)", - Self::Agenticmd => "persisting-gateway", - Self::OpenaiMsg => "dlcapt", - Self::Atif => "Harbor ATIF", - Self::Actf => "ACTF v1.0", - } - } - - pub fn primary_artifact(self) -> &'static str { - match self { - Self::Storyline => "storyline.json", - Self::Events => "events.lance", - Self::Agenticmd => "*.md", - Self::OpenaiMsg => "session_steps.json", - Self::Atif => "*.atif.json / *.atif.jsonl", - Self::Actf => "*.actf.json", - } - } -} - -impl fmt::Display for ChronicleFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ChronicleFormat { - type Err = Error; - - fn from_str(s: &str) -> Result { - match s.trim().to_ascii_lowercase().as_str() { - "storyline" => Ok(Self::Storyline), - "events" => Ok(Self::Events), - "agenticmd" => Ok(Self::Agenticmd), - "openai_msg" => Ok(Self::OpenaiMsg), - "atif" => Ok(Self::Atif), - "actf" => Ok(Self::Actf), - other => Err(Error::Other(format!( - "unknown chronicle format '{other}'; expected storyline|events|agenticmd|openai_msg|atif|actf" - ))), - } - } -} - #[cfg(test)] mod tests { use super::DocumentFormat; diff --git a/crates/persisting-pchronicle/src/formats/detect.rs b/crates/persisting-pchronicle/src/formats/detect.rs index 4d080e56..75820ff3 100644 --- a/crates/persisting-pchronicle/src/formats/detect.rs +++ b/crates/persisting-pchronicle/src/formats/detect.rs @@ -2,33 +2,30 @@ use std::path::Path; -use crate::format::ChronicleFormat; +use crate::format::DocumentFormat; use crate::Result; /// Detect format from a file path (extension / basename). /// /// `events` is detected only as a Lance dataset path (`events.lance`), never as `.json` / `.jsonl`. -pub fn detect_format_from_path(path: impl AsRef) -> Option { +pub fn detect_format_from_path(path: impl AsRef) -> Option { let path = path.as_ref(); let name = path .file_name() .and_then(|s| s.to_str()) .unwrap_or("") .to_ascii_lowercase(); - if name == "storyline.json" || name.ends_with(".storyline.json") { - return Some(ChronicleFormat::Storyline); - } if name == "events.lance" || (name.ends_with(".lance") && name.contains("event")) { - return Some(ChronicleFormat::Events); + return Some(DocumentFormat::CanonicalEvent); } - if name == "session_steps.json" || name == "session_steps.lance" { - return Some(ChronicleFormat::OpenaiMsg); + if name == "session_steps.json" { + return Some(DocumentFormat::OpenaiMsg); } if name.ends_with(".actf.json") { - return Some(ChronicleFormat::Actf); + return Some(DocumentFormat::Actf); } if name.ends_with(".md") { - return Some(ChronicleFormat::Agenticmd); + return Some(DocumentFormat::AgenticMd); } None } @@ -36,14 +33,14 @@ pub fn detect_format_from_path(path: impl AsRef) -> Option Result> { +pub fn detect_format_from_content(input: &str) -> Result> { let trimmed = input.trim_start(); if trimmed.starts_with("---") && trimmed .lines() .any(|line| line.trim() == "format: persisting") { - return Ok(Some(ChronicleFormat::Agenticmd)); + return Ok(Some(DocumentFormat::AgenticMd)); } if trimmed.starts_with('{') || trimmed.starts_with('[') { if let Ok(v) = serde_json::from_str::(trimmed) { @@ -60,12 +57,12 @@ pub fn detect_format_from_content(input: &str) -> Result } } if trimmed.contains("