From b2f5ef998d5799f870896a0786eb48d203ef2a1e Mon Sep 17 00:00:00 2001 From: Ompragash Date: Thu, 23 Jul 2026 22:50:19 +0530 Subject: [PATCH 01/44] feat: add selective workflow repair --- Cargo.lock | 2 + crates/agentctl-cli/src/main.rs | 202 ++ crates/agentctl-core/src/compiler.rs | 62 + crates/agentctl-core/src/dsl.rs | 2 + crates/agentctl-runtime/Cargo.toml | 2 + crates/agentctl-runtime/src/lib.rs | 2580 +++++++++++++++++++++++++- crates/agentctl-store/src/lib.rs | 598 +++++- fuzz/Cargo.lock | 1 + schemas/workflow.schema.json | 3 +- 9 files changed, 3409 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef06206..1f22d72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,7 +103,9 @@ dependencies = [ "async-trait", "chrono", "hex", + "jsonschema", "nix", + "rusqlite", "serde", "serde_json", "sha2", diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index 33b8a8f..f704ebc 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -83,10 +83,14 @@ enum Command { Replay(RunIdArgs), /// Create a new run from a prior workflow with fresh effects. Fork(ForkArgs), + /// Create a new run that reuses compatible upstream results and executes a repaired suffix. + Repair(RepairArgs), /// Durably request cancellation. Cancel(RunIdArgs), /// Inspect durable run, task, and audit state. Inspect(RunIdArgs), + /// Inspect or narrowly reconcile uncertain effects. + Effects(EffectArgs), /// List or resolve durable approval requests. Approvals(ApprovalArgs), /// Inspect provider capabilities or run the opt-in OpenAI smoke. @@ -178,6 +182,30 @@ struct ForkArgs { timeout_seconds: Option, } +#[derive(Debug, Args)] +struct RepairArgs { + file: PathBuf, + source_run_id: String, + #[arg(long = "from", required = true)] + from: Vec, + #[arg(long)] + plan: bool, + #[arg(long)] + restart_successful: bool, + #[arg(long)] + reason: Option, + #[arg(long, default_value = ".agentctl/runtime.db")] + db: PathBuf, + #[arg(long)] + interactive: bool, + #[arg(long)] + diff: bool, + #[arg(long)] + workspace: Option, + #[arg(long)] + timeout_seconds: Option, +} + #[derive(Debug, Args)] struct ApprovalArgs { #[arg(long, default_value = ".agentctl/runtime.db")] @@ -186,6 +214,37 @@ struct ApprovalArgs { command: ApprovalCommand, } +#[derive(Debug, Args)] +struct EffectArgs { + #[arg(long, default_value = ".agentctl/runtime.db")] + db: PathBuf, + #[command(subcommand)] + command: EffectCommand, +} + +#[derive(Debug, Subcommand)] +enum EffectCommand { + Inspect { + run_id: String, + #[arg(long)] + task: Option, + }, + Reconcile { + effect_id: String, + #[arg(long, value_enum)] + outcome: ReconciledOutcome, + #[arg(long, default_value = "cli-user")] + actor: String, + #[arg(long)] + reason: String, + }, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum ReconciledOutcome { + NotApplied, +} + #[derive(Debug, Subcommand)] enum ApprovalCommand { List { run_id: String }, @@ -515,6 +574,7 @@ async fn execute(cli: Cli) -> Result { .map_err(map_runtime_error)?; print_outcome(output, &outcome) } + Command::Repair(args) => repair_workflow(output, args).await, Command::Cancel(args) => { let store = open_store(&args.db)?; store @@ -582,6 +642,7 @@ async fn execute(cli: Cli) -> Result { print_value(output, "RunInspection", &value, Vec::new(), human)?; Ok(EXIT_OK) } + Command::Effects(args) => effect_command(output, args), Command::Approvals(args) => approval_command(output, args), Command::Providers(args) => provider_command(output, args).await, Command::Auth(args) => auth_command(output, args), @@ -713,6 +774,93 @@ async fn resume_run(output: OutputFormat, args: ResumeArgs) -> Result Result { + if !args.plan { + validate_interactive(args.interactive)?; + } + let (workflow, compiled, diagnostics) = load_and_compile(&args.file)?; + let default_base = args + .file + .parent() + .filter(|path| !path.as_os_str().is_empty()); + let base = resolve_base_path(args.workspace.as_deref().or(default_base))?; + let store = open_store(&args.db)?; + let planner = Runtime::new(store.clone(), &base); + let plan = planner + .plan_repair( + &args.source_run_id, + &workflow, + &compiled, + &args.from, + args.restart_successful, + ) + .map_err(map_runtime_error)?; + if args.plan || !plan.compatible { + let human = format!( + "repair plan: {}\nsource: {}\nreuse: {}\nexecute: {}\nblocked: {}", + if plan.compatible { + "compatible" + } else { + "blocked" + }, + plan.source_run_id, + plan.reused_tasks.join(", "), + plan.rerun_tasks.join(", "), + plan.blocked_reuse + .iter() + .map(|block| format!("{}: {}", block.task_id, block.message)) + .collect::>() + .join("; "), + ); + print_value(output, "RepairPlan", &plan, diagnostics, human)?; + return Ok(if plan.compatible { + EXIT_OK + } else { + EXIT_POLICY + }); + } + let registry = build_registry(&workflow, &base)?; + let runtime = Runtime::new(store, &base).with_registry(registry); + let cancellation = cancellation_token(args.timeout_seconds); + let outcome = runtime + .repair( + &workflow, + &compiled, + plan, + args.reason.as_deref(), + RunOptions { + check: false, + diff: args.diff, + interactive: args.interactive, + }, + &cancellation, + ) + .await + .map_err(map_runtime_error)?; + print_value( + output, + "RepairOutcome", + &outcome, + diagnostics, + format!( + "{} {:?} source={} reused={} executed={} artifacts={} trace={}", + outcome.run_id, + outcome.state, + outcome.source_run_id, + outcome.reused_tasks.join(","), + outcome.executed_tasks.join(","), + outcome + .artifacts + .iter() + .map(|artifact| artifact.path.as_str()) + .collect::>() + .join(","), + outcome.trace_id, + ), + )?; + Ok(outcome_exit_code(outcome.state)) +} + fn print_outcome( output: OutputFormat, outcome: &agentctl_runtime::RunOutcome, @@ -796,6 +944,57 @@ fn approval_command(output: OutputFormat, args: ApprovalArgs) -> Result Result { + let store = open_store(&args.db)?; + match args.command { + EffectCommand::Inspect { run_id, task } => { + let effects = store + .list_effects(&run_id) + .map_err(CliError::persistence)? + .into_iter() + .filter(|effect| { + task.as_ref() + .is_none_or(|task| effect.request.task_id == *task) + }) + .collect::>(); + print_value( + output, + "EffectInspection", + &serde_json::json!({ + "runId": run_id, + "taskId": task, + "effects": effects, + }), + Vec::new(), + format!("{} effect(s)", effects.len()), + )?; + } + EffectCommand::Reconcile { + effect_id, + outcome: ReconciledOutcome::NotApplied, + actor, + reason, + } => { + store + .reconcile_effect_not_applied(&effect_id, &actor, &reason, Utc::now()) + .map_err(CliError::persistence)?; + print_value( + output, + "EffectReconciliation", + &serde_json::json!({ + "effectId": effect_id, + "outcome": "not_applied", + "actor": actor, + "reason": reason, + }), + Vec::new(), + "effect reconciled as not applied; a compatible repair may now retry it".to_owned(), + )?; + } + } + Ok(EXIT_OK) +} + async fn provider_command(output: OutputFormat, args: ProviderArgs) -> Result { match args.command { ProviderCommand::Inspect(args) => { @@ -1609,6 +1808,7 @@ fn map_runtime_error(error: agentctl_runtime::RuntimeError) -> CliError { agentctl_runtime::RuntimeError::Provider(_) => EXIT_REMOTE, agentctl_runtime::RuntimeError::Cancelled => EXIT_CANCELLED, agentctl_runtime::RuntimeError::UncertainEffect { .. } => EXIT_POLICY, + agentctl_runtime::RuntimeError::RepairBlocked { .. } => EXIT_POLICY, _ => EXIT_RUN_FAILED, }; CliError { @@ -1640,8 +1840,10 @@ mod tests { "resume", "replay", "fork", + "repair", "cancel", "inspect", + "effects", "approvals", "providers", "auth", diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index 422f8fd..6a8d7ec 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -32,6 +32,7 @@ pub struct CompiledTask { pub retry: RetryDefinition, pub timeout_seconds: u64, pub failure: crate::dsl::FailureBehavior, + pub output_schema: Option, pub predictability: PlanPredictability, } @@ -211,6 +212,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result, #[serde(default)] pub failure: FailureBehavior, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_schema: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/agentctl-runtime/Cargo.toml b/crates/agentctl-runtime/Cargo.toml index 9561915..f94fe0c 100644 --- a/crates/agentctl-runtime/Cargo.toml +++ b/crates/agentctl-runtime/Cargo.toml @@ -16,6 +16,7 @@ agentctl-store = { version = "0.2.0", path = "../agentctl-store" } async-trait.workspace = true chrono.workspace = true hex.workspace = true +jsonschema.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true @@ -29,6 +30,7 @@ url.workspace = true nix.workspace = true [dev-dependencies] +rusqlite.workspace = true tempfile.workspace = true [lints] diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index 95e650d..e110d7b 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -1,6 +1,6 @@ //! Durable deterministic workflow runtime for agentctl. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -10,7 +10,9 @@ use agentctl_core::dsl::{ API_VERSION, ActionDefinition, ActionKind, ApprovalRequirement, EffectClass, FailureBehavior, Idempotency, Risk, ToolDefinition, ToolKind, Workflow, }; -use agentctl_core::effect::{ActionResult, ChangeStatus, EffectRequest, EffectStatus}; +use agentctl_core::effect::{ + ActionResult, ChangeStatus, EffectRecord, EffectRequest, EffectStatus, +}; use agentctl_core::policy::{PolicyContext, PolicyDecision, PolicyEngine, PolicyError, redact}; use agentctl_core::provider::{ ContentBlock, FinishReason, Message, ModelProvider, ProviderError, ProviderRequest, @@ -20,7 +22,10 @@ use agentctl_core::state::{RunState, TaskState}; use agentctl_core::template::{EvalContext, TemplateError, evaluate_when, render}; use agentctl_core::tool::{ToolContract, ToolContractError, ToolExecutor}; use agentctl_observability::{NoopTraceSink, SpanKind, TraceEvent, TracePhase, TraceSink}; -use agentctl_store::{ApprovalRequest, RunMode, SqliteStore, StoreError, TaskRecord}; +use agentctl_store::{ + ApprovalRequest, ArtifactRecord, ReusedTaskMaterialization, RunMode, SqliteStore, StoreError, + TaskCompletionMetadata, TaskDisposition, TaskExecutionMetadata, TaskRecord, +}; use async_trait::async_trait; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -220,6 +225,90 @@ pub struct RunOutcome { pub output: Option, } +pub const REPAIR_PLAN_VERSION: &str = "agentctl.dev/repair-plan/v1"; +pub const TASK_METADATA_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlannedDisposition { + Reuse, + Execute, + Removed, + Blocked, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RepairTaskPlan { + pub task_id: String, + pub disposition: PlannedDisposition, + pub reason: String, + pub source_state: Option, + pub source_fingerprint: Option, + pub target_fingerprint: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RepairBlock { + pub task_id: String, + pub rule: String, + pub message: String, + pub source_fingerprint: Option, + pub target_fingerprint: Option, + pub suggested_repair_roots: Vec, + pub full_fork_required: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FreshEffectSummary { + pub provider_tasks: usize, + pub action_tasks: usize, + pub declared_effects: usize, + pub uncertain_source_effects: usize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RepairPlan { + pub api_version: String, + pub compatible: bool, + pub source_run_id: String, + pub source_workflow_digest: String, + pub target_workflow_digest: String, + pub repair_roots: Vec, + pub restart_successful: bool, + pub reused_tasks: Vec, + pub rerun_tasks: Vec, + pub new_tasks: Vec, + pub removed_tasks: Vec, + pub changed_tasks: Vec, + pub blocked_reuse: Vec, + pub fresh_effect_summary: FreshEffectSummary, + pub approval_summary: Vec, + pub estimated_provider_tasks: usize, + pub warnings: Vec, + pub tasks: Vec, + #[serde(skip)] + materialized_tasks: Vec, + #[serde(skip)] + reconstructed_memory: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RepairOutcome { + pub run_id: String, + pub source_run_id: String, + pub trace_id: String, + pub state: RunState, + pub reused_tasks: Vec, + pub executed_tasks: Vec, + pub artifacts: Vec, + pub output: Option, +} + #[derive(Debug, Error)] pub enum RuntimeError { #[error(transparent)] @@ -259,6 +348,8 @@ pub enum RuntimeError { Io(#[from] std::io::Error), #[error("JSON error: {0}")] Json(#[from] serde_json::Error), + #[error("repair from run `{source_run_id}` is blocked by {count} compatibility rule(s)")] + RepairBlocked { source_run_id: String, count: usize }, } pub struct Runtime { @@ -529,6 +620,12 @@ impl Runtime { self.clock.now(), &trace_id, )?; + self.store.record_replayed_task_metadata( + &replay_id, + &task, + self.clock.now(), + &trace_id, + )?; } self.store.update_run_state( &replay_id, @@ -571,6 +668,811 @@ impl Runtime { self.drive(&run_id, &trace_id, options, cancellation).await } + pub fn plan_repair( + &self, + source_run_id: &str, + target_workflow: &Workflow, + target_plan: &CompiledPlan, + repair_roots: &[String], + restart_successful: bool, + ) -> Result { + let source = self.store.load_run(source_run_id)?; + if !source.state.is_terminal() { + return Err(RuntimeError::InvalidState(format!( + "repair source run `{source_run_id}` is not terminal ({:?})", + source.state + ))); + } + if repair_roots.is_empty() { + return Err(RuntimeError::InvalidState( + "repair requires at least one --from task".to_owned(), + )); + } + let source_workflow: Workflow = serde_json::from_value(source.workflow.clone())?; + + let roots = repair_roots.iter().cloned().collect::>(); + let source_tasks = self + .store + .list_tasks(source_run_id)? + .into_iter() + .map(|task| (task.task_id.clone(), task)) + .collect::>(); + let source_effects = self.store.list_effects(source_run_id)?; + let source_ids = source.plan.order.iter().cloned().collect::>(); + let target_ids = target_plan.order.iter().cloned().collect::>(); + let new_tasks = target_ids + .difference(&source_ids) + .cloned() + .collect::>(); + let removed_tasks = source_ids + .difference(&target_ids) + .cloned() + .collect::>(); + let mut rerun = roots + .iter() + .filter(|root| target_plan.tasks.contains_key(*root)) + .cloned() + .collect::>(); + loop { + let mut changed = false; + for task in target_plan.tasks.values() { + if task + .needs + .iter() + .any(|dependency| rerun.contains(dependency)) + && rerun.insert(task.id.clone()) + { + changed = true; + } + } + if !changed { + break; + } + } + + let mut blocks = Vec::new(); + if source.mode == RunMode::Replay { + blocks.push(repair_block( + "$workflow", + "recorded_replay_has_no_direct_effect_history", + "a recorded replay cannot be a repair source because its task effects were not dispatched in that run; select the original terminal run".to_owned(), + None, + None, + vec![], + true, + )); + } + if source_workflow.metadata.name != target_workflow.metadata.name { + blocks.push(repair_block( + "$workflow", + "workflow_identity_mismatch", + format!( + "source workflow `{}` and target workflow `{}` have different identities; use a full fork for unrelated workflows", + source_workflow.metadata.name, target_workflow.metadata.name + ), + None, + None, + vec![], + true, + )); + } + for root in &roots { + let Some(_) = target_plan.tasks.get(root) else { + blocks.push(repair_block( + root, + "repair_root_missing", + format!("repair root `{root}` does not exist in the target workflow"), + None, + None, + vec![], + false, + )); + continue; + }; + if source_tasks + .get(root) + .is_some_and(|task| task.state == TaskState::Succeeded) + && !restart_successful + { + blocks.push(repair_block( + root, + "successful_root_requires_acknowledgement", + format!( + "task `{root}` succeeded in the source run; use --restart-successful to execute it again" + ), + source_tasks + .get(root) + .and_then(|task| task.definition_fingerprint.clone()), + None, + vec![root.clone()], + false, + )); + } + if source_tasks.get(root).is_some_and(|task| { + task.state == TaskState::Succeeded + && matches!( + task.disposition, + TaskDisposition::Reused | TaskDisposition::Recorded + ) + }) { + blocks.push(repair_block( + root, + "indirect_effect_history", + format!( + "task `{root}` was materialized from another run and cannot be restarted without its direct effect history; select the originating run or perform a full fork" + ), + source_tasks + .get(root) + .and_then(|task| task.definition_fingerprint.clone()), + None, + vec![], + true, + )); + } + } + + for effect in source_effects + .iter() + .filter(|effect| rerun.contains(&effect.request.task_id)) + { + if repair_effect_is_unsafe(effect) { + let task_id = effect.request.task_id.clone(); + blocks.push(repair_block( + &task_id, + "unreconciled_effect", + format!( + "effect `{}` ({:?}, {:?}, {:?}) may be duplicated by repair; reconcile it before retrying", + effect.request.id, + effect.request.effect_class, + effect.request.idempotency, + effect.status + ), + None, + None, + vec![task_id.clone()], + false, + )); + } + } + + let target_policy = + PolicyEngine::new(target_workflow.spec.policy.clone(), &self.base_path)?; + let mut memory = Value::Object( + target_workflow + .spec + .memory + .working + .clone() + .into_iter() + .collect(), + ); + let inputs = source + .inputs + .as_object() + .ok_or_else(|| RuntimeError::InvalidState("run inputs must be an object".to_owned()))?; + let mut outputs = BTreeMap::new(); + let mut reused = Vec::new(); + let mut task_plans = Vec::new(); + let mut changed_tasks = BTreeSet::new(); + let mut blocked_task_ids = BTreeSet::new(); + + for task_id in &target_plan.order { + let target_task = target_plan.tasks.get(task_id).ok_or_else(|| { + RuntimeError::InvalidState(format!("target task `{task_id}` is missing")) + })?; + let target_fingerprint = + task_definition_fingerprint(target_workflow, target_task, &target_policy, None)?; + let source_task = source_tasks.get(task_id); + if source_task + .and_then(|task| task.definition_fingerprint.as_deref()) + .is_some_and(|fingerprint| fingerprint != target_fingerprint) + { + changed_tasks.insert(task_id.clone()); + } + + if rerun.contains(task_id) { + task_plans.push(RepairTaskPlan { + task_id: task_id.clone(), + disposition: PlannedDisposition::Execute, + reason: if roots.contains(task_id) { + "selected repair root".to_owned() + } else { + "transitive descendant of a repair root".to_owned() + }, + source_state: source_task.map(|task| task.state), + source_fingerprint: source_task + .and_then(|task| task.definition_fingerprint.clone()), + target_fingerprint: Some(target_fingerprint), + }); + continue; + } + + let Some(source_task) = source_task else { + let block = repair_block( + task_id, + "new_task_outside_repair_closure", + format!( + "new target task `{task_id}` is outside the repair closure and cannot be reused" + ), + None, + Some(target_fingerprint.clone()), + vec![task_id.clone()], + false, + ); + blocks.push(block); + blocked_task_ids.insert(task_id.clone()); + task_plans.push(blocked_task_plan(task_id, None, None, target_fingerprint)); + continue; + }; + let source_fingerprint = source_task.definition_fingerprint.clone(); + let blocked = |rule: &str, + message: String, + full_fork_required: bool, + blocks: &mut Vec, + blocked_task_ids: &mut BTreeSet| { + blocks.push(repair_block( + task_id, + rule, + message, + source_fingerprint.clone(), + Some(target_fingerprint.clone()), + vec![task_id.clone()], + full_fork_required, + )); + blocked_task_ids.insert(task_id.clone()); + }; + + if source_task.state != TaskState::Succeeded { + blocked( + "source_task_not_successful", + format!( + "task `{task_id}` is {:?} in the source run and has no reusable successful result", + source_task.state + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + if source_task.metadata_version != Some(TASK_METADATA_VERSION) { + blocked( + "legacy_task_metadata", + format!( + "task `{task_id}` predates repair metadata version {TASK_METADATA_VERSION}; choose it as an earlier repair root or perform a full fork" + ), + true, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + let contract = task_output_schema(target_workflow, target_task) + .unwrap_or_else(|| serde_json::json!({})); + let contract_fingerprint = versioned_json_digest(&contract)?; + if source_task.output_contract_fingerprint.as_deref() != Some(&contract_fingerprint) { + blocked( + "output_contract_mismatch", + format!("output contract for task `{task_id}` changed"), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + if source_task.definition_fingerprint.as_deref() != Some(&target_fingerprint) { + blocked( + "definition_fingerprint_mismatch", + format!( + "task `{task_id}` changed outside the repair closure; choose it as an earlier repair root" + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + let source_needs = source + .plan + .tasks + .get(task_id) + .map(|task| task.needs.as_slice()) + .unwrap_or_default(); + if source_needs != target_task.needs { + blocked( + "dependency_set_mismatch", + format!( + "task `{task_id}` has a different dependency set in the target workflow" + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + if matches!(target_task.uses, TaskUse::Agent(_)) + && task_output_schema(target_workflow, target_task).is_none() + && target_plan + .tasks + .values() + .any(|candidate| candidate.needs.contains(task_id)) + { + blocked( + "missing_output_contract", + format!( + "reused agent task `{task_id}` feeds another task but has no outputSchema or structuredOutput contract" + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + let output = source_task.output.as_ref().ok_or_else(|| { + RuntimeError::InvalidState(format!( + "successful source task `{task_id}` has no output" + )) + })?; + let output_digest = versioned_json_digest(output)?; + if source_task.output_digest.as_deref() != Some(&output_digest) { + blocked( + "output_digest_mismatch", + format!("stored output for task `{task_id}` is corrupt or was modified"), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + if let Err(message) = validate_output_contract(&contract, output) { + blocked( + "output_contract_validation", + format!("stored output for task `{task_id}` is invalid: {message}"), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + if let Err(message) = verify_artifacts(&target_policy, &source_task.artifact_manifest) { + blocked( + "artifact_integrity", + format!("artifact verification failed for task `{task_id}`: {message}"), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + let input_digest = resolved_input_digest(inputs, &memory, &outputs, target_task)?; + if source_task.input_digest.as_deref() != Some(&input_digest) { + blocked( + "resolved_input_digest_mismatch", + format!( + "resolved inputs or boundary memory for task `{task_id}` differ from the source run" + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + let state_delta = source_task.state_delta.as_ref().ok_or_else(|| { + RuntimeError::InvalidState(format!( + "successful source task `{task_id}` has no state delta" + )) + })?; + let state_delta_digest = versioned_json_digest(state_delta)?; + if source_task.state_delta_digest.as_deref() != Some(&state_delta_digest) { + blocked( + "state_delta_digest_mismatch", + format!("state delta for task `{task_id}` is corrupt or unsupported"), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + apply_state_delta(&mut memory, state_delta)?; + outputs.insert(task_id.clone(), output.clone()); + let source_effect_summary = if source_task.disposition == TaskDisposition::Reused { + source_task + .reuse_decision + .as_ref() + .and_then(|decision| decision.get("sourceEffects")) + .cloned() + .unwrap_or_else(|| serde_json::json!([])) + } else { + Value::Array( + source_effects + .iter() + .filter(|effect| effect.request.task_id == *task_id) + .map(|effect| { + serde_json::json!({ + "effectId": effect.request.id, + "effectClass": effect.request.effect_class, + "idempotency": effect.request.idempotency, + "status": effect.status, + "confirmed": effect.confirmed, + }) + }) + .collect(), + ) + }; + let reuse_decision = serde_json::json!({ + "formatVersion": 1, + "reason": "all compatibility checks passed", + "checks": [ + "source_succeeded", + "outside_rerun_closure", + "definition_fingerprint", + "resolved_input_digest", + "dependency_set", + "output_contract", + "output_digest", + "artifact_integrity", + "state_delta_digest", + "effect_certainty" + ], + "sourceWorkflowDigest": source.workflow_digest, + "targetWorkflowDigest": target_plan.workflow_digest, + "sourceEffects": source_effect_summary, + }); + reused.push(ReusedTaskMaterialization { + task_id: task_id.clone(), + source_run_id: source_run_id.to_owned(), + source_task_id: task_id.clone(), + source_attempt: source_task.attempt, + output: output.clone(), + metadata: TaskCompletionMetadata { + execution: TaskExecutionMetadata { + metadata_version: TASK_METADATA_VERSION, + definition_fingerprint: target_fingerprint.clone(), + input_digest, + output_contract_fingerprint: contract_fingerprint, + }, + output_digest, + state_delta: state_delta.clone(), + state_delta_digest, + artifact_manifest: source_task.artifact_manifest.clone(), + }, + reuse_decision, + }); + task_plans.push(RepairTaskPlan { + task_id: task_id.clone(), + disposition: PlannedDisposition::Reuse, + reason: "successful compatible source result".to_owned(), + source_state: Some(source_task.state), + source_fingerprint, + target_fingerprint: Some(target_fingerprint), + }); + } + + for task_id in target_plan + .order + .iter() + .filter(|task_id| rerun.contains(*task_id)) + { + let target_task = target_plan.tasks.get(task_id).ok_or_else(|| { + RuntimeError::InvalidState(format!("target task `{task_id}` is missing")) + })?; + if !target_task + .needs + .iter() + .all(|dependency| outputs.contains_key(dependency)) + { + continue; + } + if let Err(error) = resolved_input_digest(inputs, &memory, &outputs, target_task) { + let source_fingerprint = source_tasks + .get(task_id) + .and_then(|task| task.definition_fingerprint.clone()); + let target_fingerprint = task_plans + .iter() + .find(|task| &task.task_id == task_id) + .and_then(|task| task.target_fingerprint.clone()); + blocks.push(repair_block( + task_id, + "target_input_resolution", + format!( + "target task `{task_id}` cannot consume the reconstructed boundary state: {error}" + ), + source_fingerprint, + target_fingerprint, + vec![task_id.clone()], + false, + )); + blocked_task_ids.insert(task_id.clone()); + } + } + + for task_id in &removed_tasks { + task_plans.push(RepairTaskPlan { + task_id: task_id.clone(), + disposition: PlannedDisposition::Removed, + reason: "task is absent from the target workflow".to_owned(), + source_state: source_tasks.get(task_id).map(|task| task.state), + source_fingerprint: source_tasks + .get(task_id) + .and_then(|task| task.definition_fingerprint.clone()), + target_fingerprint: None, + }); + } + for task_plan in &mut task_plans { + if blocked_task_ids.contains(&task_plan.task_id) { + task_plan.disposition = PlannedDisposition::Blocked; + } + } + + let rerun_tasks = target_plan + .order + .iter() + .filter(|task| rerun.contains(*task)) + .cloned() + .collect::>(); + let reused_tasks = target_plan + .order + .iter() + .filter(|task| { + reused + .iter() + .any(|materialized| materialized.task_id.as_str() == task.as_str()) + }) + .cloned() + .collect::>(); + let provider_tasks = rerun_tasks + .iter() + .filter(|task| { + target_plan + .tasks + .get(*task) + .is_some_and(|task| matches!(task.uses, TaskUse::Agent(_))) + }) + .count(); + let action_tasks = rerun_tasks.len().saturating_sub(provider_tasks); + let declared_effects = target_plan + .requirements + .effects + .iter() + .filter(|effect| rerun.contains(&effect.task)) + .count(); + let approval_summary = target_plan + .requirements + .effects + .iter() + .filter(|effect| rerun.contains(&effect.task) && effect.approval_possible) + .map(|effect| format!("{}:{}", effect.task, effect.operation)) + .collect::>(); + let uncertain_source_effects = source_effects + .iter() + .filter(|effect| { + rerun.contains(&effect.request.task_id) + && matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) + }) + .count(); + let compatible = blocks.is_empty(); + Ok(RepairPlan { + api_version: REPAIR_PLAN_VERSION.to_owned(), + compatible, + source_run_id: source_run_id.to_owned(), + source_workflow_digest: source.workflow_digest, + target_workflow_digest: target_plan.workflow_digest.clone(), + repair_roots: roots.into_iter().collect(), + restart_successful, + reused_tasks, + rerun_tasks, + new_tasks, + removed_tasks, + changed_tasks: changed_tasks.into_iter().collect(), + blocked_reuse: blocks, + fresh_effect_summary: FreshEffectSummary { + provider_tasks, + action_tasks, + declared_effects, + uncertain_source_effects, + }, + approval_summary, + estimated_provider_tasks: provider_tasks, + warnings: vec![ + "repair roots and descendants execute with fresh effects".to_owned(), + "reused tasks dispatch no providers, tools, processes, or network calls".to_owned(), + ], + tasks: task_plans, + materialized_tasks: reused, + reconstructed_memory: memory, + }) + } + + #[allow(clippy::too_many_arguments)] + pub async fn repair( + &self, + target_workflow: &Workflow, + target_plan: &CompiledPlan, + plan: RepairPlan, + reason: Option<&str>, + options: RunOptions, + cancellation: &CancellationToken, + ) -> Result { + if !plan.compatible { + return Err(RuntimeError::RepairBlocked { + source_run_id: plan.source_run_id, + count: plan.blocked_reuse.len(), + }); + } + if target_plan.workflow_digest != plan.target_workflow_digest { + return Err(RuntimeError::InvalidState( + "target workflow changed after repair planning; create a new repair plan" + .to_owned(), + )); + } + let plan = self.plan_repair( + &plan.source_run_id, + target_workflow, + target_plan, + &plan.repair_roots, + plan.restart_successful, + )?; + if !plan.compatible { + return Err(RuntimeError::RepairBlocked { + source_run_id: plan.source_run_id, + count: plan.blocked_reuse.len(), + }); + } + let source = self.store.load_run(&plan.source_run_id)?; + let run_id = self.ids.next_id("repair"); + let trace_id = self.ids.next_id("trace"); + self.store.create_repair_run( + &run_id, + &plan.source_run_id, + &plan.source_workflow_digest, + API_VERSION, + &serde_json::to_value(target_workflow)?, + target_plan, + &source.inputs, + &plan.reconstructed_memory, + &plan.repair_roots, + reason, + &plan.materialized_tasks, + &serde_json::to_value(&plan.tasks)?, + &self.base_path, + self.clock.now(), + &trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Run, + TracePhase::Started, + "run.repair", + &trace_id, + &run_id, + self.clock.now(), + ) + .attributes( + serde_json::json!({ + "sourceRunId": plan.source_run_id, + "repairRoots": plan.repair_roots, + "reusedTasks": plan.reused_tasks, + "executedTasks": plan.rerun_tasks, + }), + &[], + ), + )?; + for task in &plan.materialized_tasks { + self.trace( + TraceEvent::new( + SpanKind::Task, + TracePhase::Completed, + "task.reused", + &trace_id, + &run_id, + self.clock.now(), + ) + .task(&task.task_id) + .attributes( + serde_json::json!({ + "disposition": "reused", + "sourceRunId": task.source_run_id, + "sourceTaskId": task.source_task_id, + "sourceAttempt": task.source_attempt, + "outputDigest": task.metadata.output_digest, + }), + &[], + ), + )?; + } + let outcome = self + .drive(&run_id, &trace_id, options, cancellation) + .await?; + let artifacts = self + .store + .list_tasks(&outcome.run_id)? + .into_iter() + .flat_map(|task| task.artifact_manifest) + .map(|artifact| (artifact.path.clone(), artifact)) + .collect::>() + .into_values() + .collect(); + Ok(RepairOutcome { + run_id: outcome.run_id, + source_run_id: source.run_id, + trace_id: outcome.trace_id, + state: outcome.state, + reused_tasks: plan.reused_tasks, + executed_tasks: plan.rerun_tasks, + artifacts, + output: outcome.output, + }) + } + async fn drive( &self, run_id: &str, @@ -728,11 +1630,46 @@ impl Runtime { .into_iter() .find(|record| record.task_id == task.id) .ok_or_else(|| RuntimeError::InvalidState(format!("task `{}` missing", task.id)))?; - let execution = self - .execute_task( - &workflow, - &run, - ¤t, + let task_outputs = tasks + .iter() + .filter_map(|record| { + record + .output + .clone() + .map(|output| (record.task_id.clone(), output)) + }) + .collect::>(); + let execution_contract = if options.check { + serde_json::json!({}) + } else { + task_output_schema(&workflow, task).unwrap_or_else(|| serde_json::json!({})) + }; + let execution_metadata = TaskExecutionMetadata { + metadata_version: TASK_METADATA_VERSION, + definition_fingerprint: task_definition_fingerprint( + &workflow, task, &policy, None, + )?, + input_digest: resolved_input_digest( + run.inputs.as_object().ok_or_else(|| { + RuntimeError::InvalidState("run inputs must be an object".to_owned()) + })?, + &run.working_memory, + &task_outputs, + task, + )?, + output_contract_fingerprint: versioned_json_digest(&execution_contract)?, + }; + self.store.record_task_execution_metadata( + run_id, + &task.id, + &execution_metadata, + self.clock.now(), + )?; + let execution = self + .execute_task( + &workflow, + &run, + ¤t, task, &policy, trace_id, @@ -740,15 +1677,42 @@ impl Runtime { cancellation, ) .await; + let execution = execution.and_then(|execution| { + if let TaskExecution::Complete { output, .. } = &execution { + validate_output_contract(&execution_contract, output).map_err(|message| { + RuntimeError::Task { + task: task.id.clone(), + message: format!("task output contract failed: {message}"), + } + })?; + } + Ok(execution) + }); match execution { Ok(TaskExecution::Complete { output, memory }) => { - self.store.transition_task( + let effects = self.store.list_effects(run_id)?; + let delta = state_delta(&run.working_memory, memory.as_ref())?; + let completion = TaskCompletionMetadata { + execution: TaskExecutionMetadata { + definition_fingerprint: task_definition_fingerprint( + &workflow, + task, + &policy, + Some(&effects), + )?, + ..execution_metadata + }, + output_digest: versioned_json_digest(&output)?, + state_delta_digest: versioned_json_digest(&delta)?, + artifact_manifest: collect_artifacts(&policy, &effects, &task.id)?, + state_delta: delta, + }; + self.store.complete_task( run_id, &task.id, - TaskState::Succeeded, - Some(&output), - None, + &output, memory.as_ref(), + &completion, self.clock.now(), trace_id, )?; @@ -1971,8 +2935,30 @@ impl Runtime { } match response.finish_reason { FinishReason::Complete => { + let output = if let Some(schema) = &agent.structured_output { + let structured: Value = + serde_json::from_str(&response.text).map_err(|error| { + RuntimeError::Task { + task: task.task_id.clone(), + message: format!( + "provider structured output was not valid JSON: {error}" + ), + } + })?; + validate_output_contract(schema, &structured).map_err(|message| { + RuntimeError::Task { + task: task.task_id.clone(), + message: format!( + "provider structured output failed its contract: {message}" + ), + } + })?; + structured + } else { + serde_json::json!({"text": response.text, "usage": usage}) + }; return Ok(TaskExecution::Complete { - output: serde_json::json!({"text": response.text, "usage": usage}), + output, memory: None, }); } @@ -2212,6 +3198,376 @@ enum TaskExecution { Paused, } +fn repair_block( + task_id: &str, + rule: &str, + message: String, + source_fingerprint: Option, + target_fingerprint: Option, + suggested_repair_roots: Vec, + full_fork_required: bool, +) -> RepairBlock { + RepairBlock { + task_id: task_id.to_owned(), + rule: rule.to_owned(), + message, + source_fingerprint, + target_fingerprint, + suggested_repair_roots, + full_fork_required, + } +} + +fn blocked_task_plan( + task_id: &str, + source_state: Option, + source_fingerprint: Option, + target_fingerprint: String, +) -> RepairTaskPlan { + RepairTaskPlan { + task_id: task_id.to_owned(), + disposition: PlannedDisposition::Blocked, + reason: "reuse compatibility check failed".to_owned(), + source_state, + source_fingerprint, + target_fingerprint: Some(target_fingerprint), + } +} + +fn repair_effect_is_unsafe(effect: &EffectRecord) -> bool { + let potentially_mutating = matches!( + effect.request.effect_class, + EffectClass::WorkspaceMutate + | EffectClass::ExternalMutate + | EffectClass::ProcessExecution + | EffectClass::Network + | EffectClass::RemoteAgent + ); + potentially_mutating + && (matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) || (effect.status == EffectStatus::Succeeded + && matches!( + effect.request.idempotency, + Idempotency::AtMostOnce | Idempotency::Unknown + ))) +} + +fn task_output_schema(workflow: &Workflow, task: &agentctl_core::CompiledTask) -> Option { + task.output_schema.clone().or_else(|| match &task.uses { + TaskUse::Agent(name) => workflow + .spec + .agents + .get(name) + .and_then(|agent| agent.structured_output.clone()), + TaskUse::Action(_) => Some(serde_json::json!({"type": "object"})), + }) +} + +fn validate_output_contract(schema: &Value, output: &Value) -> Result<(), String> { + let validator = jsonschema::validator_for(schema).map_err(|error| error.to_string())?; + validator + .validate(output) + .map_err(|error| error.to_string()) +} + +fn task_definition_fingerprint( + workflow: &Workflow, + task: &agentctl_core::CompiledTask, + policy: &PolicyEngine, + recorded_effects: Option<&[EffectRecord]>, +) -> Result { + let execution = match &task.uses { + TaskUse::Action(name) => serde_json::json!({ + "kind": "action", + "task": task, + "action": workflow.spec.actions.get(name), + }), + TaskUse::Agent(name) => { + let agent = workflow.spec.agents.get(name).ok_or_else(|| { + RuntimeError::InvalidState(format!("agent `{name}` disappeared after compile")) + })?; + let provider = workflow + .spec + .providers + .get(&agent.provider) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "provider `{}` disappeared after compile", + agent.provider + )) + })?; + let tools = agent + .tools + .iter() + .map(|tool| { + workflow + .spec + .tools + .get(tool) + .map(|definition| (tool.clone(), definition.clone())) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "tool `{tool}` disappeared after compile" + )) + }) + }) + .collect::, _>>()?; + let instruction_content_digest = if let Some(path) = &agent.instructions_file { + let recorded = recorded_effects.and_then(|effects| { + effects + .iter() + .rev() + .find(|effect| { + effect.request.task_id == task.id + && effect.request.operation == "agent.instructions.read" + && effect.status == EffectStatus::Succeeded + && effect.confirmed + }) + .and_then(|effect| effect.result.as_ref()) + .and_then(|result| result.get("content")) + .and_then(Value::as_str) + }); + let content = recorded.map(ToOwned::to_owned).map_or_else( + || read_bounded_text_sync(&policy.resolve_read_path(path)?), + Ok, + )?; + Some(format!("sha256:{}", digest(content.as_bytes()))) + } else { + None + }; + serde_json::json!({ + "kind": "agent", + "task": task, + "agent": agent, + "provider": provider, + "tools": tools, + "instructionContentDigest": instruction_content_digest, + }) + } + }; + versioned_json_digest(&serde_json::json!({ + "formatVersion": 1, + "execution": execution, + "policy": workflow.spec.policy, + "packs": workflow.spec.packs, + })) +} + +fn resolved_input_digest( + inputs: &serde_json::Map, + memory: &Value, + outputs: &BTreeMap, + task: &agentctl_core::CompiledTask, +) -> Result { + let memory_object = memory + .as_object() + .ok_or_else(|| RuntimeError::InvalidState("working memory must be an object".to_owned()))?; + let mut context = EvalContext { + inputs: inputs + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + vars: BTreeMap::new(), + memory: memory_object + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + tasks: outputs.clone(), + }; + context.vars = task + .vars + .iter() + .map(|(name, value)| render(value, &context).map(|value| (name.clone(), value))) + .collect::, _>>()?; + let input = render(&serde_json::to_value(&task.input)?, &context)?; + let dependencies = task + .needs + .iter() + .filter_map(|dependency| { + outputs + .get(dependency) + .map(|output| (dependency.clone(), output.clone())) + }) + .collect::>(); + versioned_json_digest(&serde_json::json!({ + "formatVersion": 1, + "input": input, + "vars": context.vars, + "workingMemory": memory, + "dependencies": dependencies, + })) +} + +fn state_delta(before: &Value, after: Option<&Value>) -> Result { + let before = before + .as_object() + .ok_or_else(|| RuntimeError::InvalidState("working memory must be an object".to_owned()))?; + let after = match after { + Some(after) => after.as_object().ok_or_else(|| { + RuntimeError::InvalidState("working memory must be an object".to_owned()) + })?, + None => before, + }; + let set = after + .iter() + .filter(|(key, value)| before.get(*key) != Some(*value)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + let remove = before + .keys() + .filter(|key| !after.contains_key(*key)) + .cloned() + .collect::>(); + Ok(serde_json::json!({ + "formatVersion": 1, + "set": set, + "remove": remove, + })) +} + +fn apply_state_delta(memory: &mut Value, delta: &Value) -> Result<(), RuntimeError> { + if delta.get("formatVersion").and_then(Value::as_u64) != Some(1) { + return Err(RuntimeError::InvalidState( + "unsupported task state-delta format".to_owned(), + )); + } + let object = memory + .as_object_mut() + .ok_or_else(|| RuntimeError::InvalidState("working memory must be an object".to_owned()))?; + let set = delta + .get("set") + .and_then(Value::as_object) + .ok_or_else(|| RuntimeError::InvalidState("state delta has no set object".to_owned()))?; + for (key, value) in set { + object.insert(key.clone(), value.clone()); + } + let remove = delta + .get("remove") + .and_then(Value::as_array) + .ok_or_else(|| RuntimeError::InvalidState("state delta has no remove array".to_owned()))?; + for key in remove { + let key = key.as_str().ok_or_else(|| { + RuntimeError::InvalidState("state delta key is not a string".to_owned()) + })?; + object.remove(key); + } + Ok(()) +} + +fn versioned_json_digest(value: &Value) -> Result { + let canonical = canonical_json(value); + Ok(format!( + "sha256:v1:{}", + digest(&serde_json::to_vec(&canonical)?) + )) +} + +fn canonical_json(value: &Value) -> Value { + match value { + Value::Object(object) => Value::Object( + object + .iter() + .map(|(key, value)| (key.clone(), canonical_json(value))) + .collect(), + ), + Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()), + other => other.clone(), + } +} + +fn read_bounded_text_sync(path: &Path) -> Result { + use std::io::Read as _; + + let file = std::fs::File::open(path)?; + let mut reader = file.take(MAX_WORKSPACE_FILE_BYTES + 1); + let mut content = String::new(); + reader.read_to_string(&mut content)?; + if content.len() as u64 > MAX_WORKSPACE_FILE_BYTES { + return Err(RuntimeError::InvalidState(format!( + "file {} exceeds {MAX_WORKSPACE_FILE_BYTES} bytes", + path.display() + ))); + } + Ok(content) +} + +fn collect_artifacts( + policy: &PolicyEngine, + effects: &[EffectRecord], + task_id: &str, +) -> Result, RuntimeError> { + let mut paths = BTreeSet::new(); + for effect in effects.iter().filter(|effect| { + effect.request.task_id == task_id + && effect.request.effect_class == EffectClass::WorkspaceMutate + && effect.status == EffectStatus::Succeeded + && effect.confirmed + }) { + if let Some(result) = &effect.result { + collect_result_paths(result, &mut paths); + } + } + paths + .into_iter() + .map(|path| { + let resolved = policy.resolve_read_path(&path)?; + let metadata = std::fs::metadata(&resolved)?; + if metadata.len() > 16 * 1024 * 1024 { + return Err(RuntimeError::InvalidState(format!( + "artifact `{path}` exceeds 16777216 bytes" + ))); + } + let content = std::fs::read(&resolved)?; + Ok(ArtifactRecord { + path, + digest: format!("sha256:{}", digest(&content)), + size_bytes: metadata.len(), + }) + }) + .collect() +} + +fn collect_result_paths(value: &Value, paths: &mut BTreeSet) { + match value { + Value::Object(object) => { + if let Some(path) = object.get("path").and_then(Value::as_str) { + paths.insert(path.to_owned()); + } + for value in object.values() { + collect_result_paths(value, paths); + } + } + Value::Array(values) => { + for value in values { + collect_result_paths(value, paths); + } + } + _ => {} + } +} + +fn verify_artifacts(policy: &PolicyEngine, artifacts: &[ArtifactRecord]) -> Result<(), String> { + for artifact in artifacts { + let resolved = policy + .resolve_read_path(&artifact.path) + .map_err(|error| error.to_string())?; + let metadata = + std::fs::metadata(&resolved).map_err(|error| format!("{}: {error}", artifact.path))?; + if metadata.len() != artifact.size_bytes { + return Err(format!("{} size mismatch", artifact.path)); + } + let content = + std::fs::read(&resolved).map_err(|error| format!("{}: {error}", artifact.path))?; + let actual = format!("sha256:{}", digest(&content)); + if actual != artifact.digest { + return Err(format!("{} digest mismatch", artifact.path)); + } + } + Ok(()) +} + fn next_task<'a>( plan: &'a CompiledPlan, records: &[TaskRecord], @@ -2401,7 +3757,8 @@ const fn retryable_error(error: &RuntimeError) -> bool { | RuntimeError::UncertainEffect { .. } | RuntimeError::ExternalEffectUncertain(_) | RuntimeError::Cancelled - | RuntimeError::Json(_) => false, + | RuntimeError::Json(_) + | RuntimeError::RepairBlocked { .. } => false, } } @@ -2454,7 +3811,7 @@ mod tests { use agentctl_core::tool::{ToolContract, ToolContractError, ToolExecutor}; use agentctl_observability::BufferedTraceSink; use agentctl_store::ApprovalResolution; - use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use tempfile::tempdir; struct FixedClock; @@ -2467,6 +3824,24 @@ mod tests { } } + struct MutableClock(AtomicI64); + + impl MutableClock { + fn new() -> Self { + Self(AtomicI64::new(1_767_225_600)) + } + + fn advance(&self, seconds: i64) { + self.0.fetch_add(seconds, Ordering::SeqCst); + } + } + + impl Clock for MutableClock { + fn now(&self) -> DateTime { + DateTime::from_timestamp(self.0.load(Ordering::SeqCst), 0).unwrap_or_else(Utc::now) + } + } + #[derive(Default)] struct SequenceIds(AtomicU64); @@ -2547,6 +3922,64 @@ mod tests { } } + #[derive(Default)] + struct SelectiveRepairProvider { + first_calls: AtomicU64, + second_calls: AtomicU64, + } + + #[async_trait] + impl ModelProvider for SelectiveRepairProvider { + fn name(&self) -> &'static str { + "fake" + } + + async fn complete( + &self, + request: &ProviderRequest, + _cancellation: &CancellationToken, + ) -> Result { + let prompt = match request.messages.first() { + Some(Message::User(content)) => content + .first() + .and_then(|block| match block { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .unwrap_or_default(), + _ => "", + }; + let text = if prompt == "produce durable output" { + assert_eq!( + self.first_calls.fetch_add(1, Ordering::SeqCst), + 0, + "reused upstream provider must not execute during repair" + ); + r#"{"value":"durable"}"#.to_owned() + } else if prompt.starts_with("broken") { + self.second_calls.fetch_add(1, Ordering::SeqCst); + "not-json".to_owned() + } else { + assert_eq!(prompt, "fixed durable"); + self.second_calls.fetch_add(1, Ordering::SeqCst); + r#"{"received":"durable"}"#.to_owned() + }; + Ok(ProviderResponse { + response_id: Some("repair-test".to_owned()), + text: text.clone(), + tool_calls: Vec::new(), + assistant_content: vec![ContentBlock::Text { text }], + continuation: None, + usage: Usage { + input_tokens: 2, + output_tokens: 1, + ..Usage::default() + }, + finish_reason: FinishReason::Complete, + }) + } + } + struct RetryableThenCancelProvider; #[async_trait] @@ -2751,46 +4184,1115 @@ mod tests { } #[tokio::test] - async fn deterministic_dataflow_condition_and_working_memory() { + async fn selective_repair_reuses_upstream_agent_output_without_dispatch() { let directory = tempdir().expect("tempdir"); let store = SqliteStore::open_memory().expect("store"); - let (workflow, plan) = compile_fixture( - r#" + let provider = Arc::new(SelectiveRepairProvider::default()); + let clock = Arc::new(MutableClock::new()); + let runtime = Runtime::new(store.clone(), directory.path()) + .with_clock(clock.clone()) + .with_ids(Arc::new(SequenceIds::default())) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let source_yaml = r#" apiVersion: agentctl.dev/v1alpha1 kind: Workflow -metadata: { name: dataflow } +metadata: { name: selective-repair } spec: - inputs: { enabled: true, greeting: hello } - memory: - working: { count: 0 } - actions: - assign: { kind: builtin.assign } - remember: { kind: builtin.memory.write } + providers: + fake: { kind: fake } + agents: + first: + provider: fake + model: fake + instructions: produce structured output + structuredOutput: + type: object + required: [value] + additionalProperties: false + properties: + value: { type: string } + second: + provider: fake + model: fake + instructions: consume structured output + structuredOutput: + type: object + required: [received] + additionalProperties: false + properties: + received: { type: string } tasks: - id: first - uses: action:assign - with: { message: "${{ inputs.greeting }}" } - - id: remember - uses: action:remember + uses: agent:first + with: { prompt: produce durable output } + - id: second + uses: agent:second needs: [first] - when: "${{ inputs.enabled == true }}" - with: { key: result, value: "${{ tasks.first.output.output.message }}" } -"#, + with: { prompt: "broken ${{ tasks.first.output.value }}" } +"#; + let repaired_yaml = source_yaml.replace( + r#"with: { prompt: "broken ${{ tasks.first.output.value }}" }"#, + r#"with: { prompt: "fixed ${{ tasks.first.output.value }}" }"#, ); - let outcome = runtime(store.clone(), directory.path()) + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime .start( - &workflow, - &plan, - serde_json::json!({"enabled": true, "greeting": "hello"}), + &source_workflow, + &source_plan, + serde_json::json!({}), RunOptions::default(), &CancellationToken::new(), ) .await - .expect("run succeeds"); + { + Err(RuntimeError::RunFailed { run_id, task, .. }) => { + assert_eq!(task, "second"); + run_id + } + other => panic!("expected source failure, got {other:?}"), + }; + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("repair plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + assert_eq!(plan.reused_tasks, ["first"]); + assert_eq!(plan.rerun_tasks, ["second"]); + + let invalid_consumer_yaml = + repaired_yaml.replace("tasks.first.output.value", "tasks.first.output.missing"); + let (invalid_consumer_workflow, invalid_consumer_plan) = + compile_fixture(&invalid_consumer_yaml); + let invalid_consumer = runtime + .plan_repair( + &source_run_id, + &invalid_consumer_workflow, + &invalid_consumer_plan, + &["second".to_owned()], + false, + ) + .expect("invalid consumer plan"); + assert!(!invalid_consumer.compatible); + assert!( + invalid_consumer.blocked_reuse.iter().any(|block| { + block.task_id == "second" && block.rule == "target_input_resolution" + }) + ); + + clock.advance(3_600); + let outcome = runtime + .repair( + &repaired_workflow, + &repaired_plan, + plan, + Some("fix second task"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair succeeds"); assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!(provider.first_calls.load(Ordering::SeqCst), 1); + assert_eq!(provider.second_calls.load(Ordering::SeqCst), 2); assert_eq!( - store.load_run(&outcome.run_id).expect("run").working_memory["result"], - "hello" + outcome.output.as_ref().expect("output")["second"]["received"], + "durable" + ); + let repaired_tasks = store.list_tasks(&outcome.run_id).expect("tasks"); + assert_eq!( + repaired_tasks[0].disposition, + agentctl_store::TaskDisposition::Reused + ); + assert_eq!( + repaired_tasks[1].disposition, + agentctl_store::TaskDisposition::Executed + ); + assert!( + repaired_tasks[0] + .reuse_decision + .as_ref() + .and_then(|decision| decision.get("sourceEffects")) + .and_then(Value::as_array) + .is_some_and(|effects| !effects.is_empty()) + ); + assert!( + store + .trace_events(&outcome.run_id) + .expect("traces") + .iter() + .any(|event| { + event.event["name"] == "task.reused" && event.event["taskId"] == "first" + }) + ); + assert!( + store + .provider_sessions(&outcome.run_id) + .expect("sessions") + .iter() + .all(|session| session.task_id != "first") + ); + assert_eq!( + store + .list_effects(&outcome.run_id) + .expect("effects") + .iter() + .filter(|effect| effect.request.task_id == "first") + .count(), + 0 + ); + assert_eq!( + store.load_run(&source_run_id).expect("source").state, + RunState::Failed + ); + let cutoff = DateTime::from_timestamp(1_767_227_400, 0).expect("cutoff"); + store.garbage_collect(cutoff).expect("source gc"); + assert!(matches!( + store.load_run(&source_run_id), + Err(StoreError::RunNotFound(_)) + )); + assert_eq!( + store + .load_run(&outcome.run_id) + .expect("repair survives") + .state, + RunState::Succeeded + ); + let replay = runtime + .replay(&outcome.run_id) + .await + .expect("offline replay"); + assert_eq!(replay.state, RunState::Succeeded); + assert_eq!(replay.output, outcome.output); + assert!( + store + .list_effects(&replay.run_id) + .expect("replay effects") + .is_empty() + ); + assert!( + store + .provider_sessions(&replay.run_id) + .expect("replay sessions") + .is_empty() + ); + let replay_source_plan = runtime + .plan_repair( + &replay.run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + true, + ) + .expect("replay-source plan"); + assert!(!replay_source_plan.compatible); + assert!( + replay_source_plan + .blocked_reuse + .iter() + .any(|block| { block.rule == "recorded_replay_has_no_direct_effect_history" }) + ); + } + + #[tokio::test] + async fn repair_plan_uses_minimal_branch_closure_and_blocks_changed_upstream() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: branch-repair } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - id: prepare + uses: action:assign + with: { value: stable } + - id: analyze_a + uses: action:assign + needs: [prepare] + with: { branch: a } + - id: analyze_b + uses: action:assert + needs: [prepare] + with: { that: false, message: broken } + - id: combine + uses: action:assign + needs: [analyze_a, analyze_b] + with: { result: combined } +"#; + let repaired_yaml = source_yaml.replace( + "with: { that: false, message: broken }", + "with: { that: true, message: fixed }", + ); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, task, .. }) => { + assert_eq!(task, "analyze_b"); + run_id + } + other => panic!("expected source failure, got {other:?}"), + }; + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["analyze_b".to_owned()], + false, + ) + .expect("plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + assert_eq!(plan.reused_tasks, ["prepare", "analyze_a"]); + assert_eq!(plan.rerun_tasks, ["analyze_b", "combine"]); + let outcome = runtime + .repair( + &repaired_workflow, + &repaired_plan, + plan, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair"); + assert_eq!(outcome.state, RunState::Succeeded); + + let changed_upstream_yaml = + repaired_yaml.replace("with: { value: stable }", "with: { value: changed }"); + let (changed_workflow, changed_plan) = compile_fixture(&changed_upstream_yaml); + let blocked = runtime + .plan_repair( + &source_run_id, + &changed_workflow, + &changed_plan, + &["analyze_b".to_owned()], + false, + ) + .expect("blocked plan"); + assert!(!blocked.compatible); + assert!(blocked.blocked_reuse.iter().any(|block| { + block.task_id == "prepare" && block.rule == "definition_fingerprint_mismatch" + })); + + let changed_contract_yaml = repaired_yaml.replace( + " with: { value: stable }", + concat!( + " with: { value: stable }\n", + " outputSchema:\n", + " type: object\n", + " properties: { value: { type: integer } }\n", + " required: [value]\n" + ), + ); + let (contract_workflow, contract_plan) = compile_fixture(&changed_contract_yaml); + let contract_blocked = runtime + .plan_repair( + &source_run_id, + &contract_workflow, + &contract_plan, + &["analyze_b".to_owned()], + false, + ) + .expect("contract plan"); + assert!(!contract_blocked.compatible); + assert!(contract_blocked.blocked_reuse.iter().any(|block| { + block.task_id == "prepare" && block.rule == "output_contract_mismatch" + })); + + let unrelated_yaml = repaired_yaml.replace( + "metadata: { name: branch-repair }", + "metadata: { name: unrelated }", + ); + let (unrelated_workflow, unrelated_plan) = compile_fixture(&unrelated_yaml); + let unrelated = runtime + .plan_repair( + &source_run_id, + &unrelated_workflow, + &unrelated_plan, + &["analyze_b".to_owned()], + false, + ) + .expect("unrelated plan"); + assert!(!unrelated.compatible); + assert!(unrelated.blocked_reuse.iter().any(|block| { + block.rule == "workflow_identity_mismatch" && block.full_fork_required + })); + } + + #[tokio::test] + async fn repair_blocks_uncertain_mutation_until_not_applied_reconciliation() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: effect-repair } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:assign + with: { value: durable } + - id: second + uses: action:assert + needs: [first] + with: { that: false } +"#; + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let effect = EffectRequest::new( + &source_run_id, + "second", + 1, + 1, + "external.publish", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"record": "x"}), + "publish a record", + "trace-uncertain", + ); + store + .record_effect_request(&effect, FixedClock.now()) + .expect("record effect"); + store + .mark_effect_started(&effect.id, FixedClock.now()) + .expect("start effect"); + store + .mark_effect_uncertain(&effect.id, "dispatch outcome unknown", FixedClock.now()) + .expect("uncertain effect"); + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let blocked = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("blocked plan"); + assert!(!blocked.compatible); + assert!( + blocked + .blocked_reuse + .iter() + .any(|block| { block.task_id == "second" && block.rule == "unreconciled_effect" }) + ); + + store + .reconcile_effect_not_applied( + &effect.id, + "operator", + "remote system confirms no record", + FixedClock.now(), + ) + .expect("reconcile"); + let read_only = EffectRequest::new( + &source_run_id, + "second", + 1, + 2, + "external.read", + EffectClass::Observe, + Risk::Low, + Idempotency::Idempotent, + serde_json::json!({"record": "x"}), + "read a record", + "trace-read-only", + ); + store + .record_effect_request(&read_only, FixedClock.now()) + .expect("record read-only effect"); + store + .mark_effect_started(&read_only.id, FixedClock.now()) + .expect("start read-only effect"); + store + .mark_effect_uncertain(&read_only.id, "read response was lost", FixedClock.now()) + .expect("uncertain read-only effect"); + let compatible = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("compatible plan"); + assert!(compatible.compatible, "{:?}", compatible.blocked_reuse); + assert_eq!(compatible.fresh_effect_summary.uncertain_source_effects, 1); + } + + #[tokio::test] + async fn repair_detects_changed_upstream_prompt_file() { + let directory = tempdir().expect("tempdir"); + std::fs::write(directory.path().join("first.txt"), "original instructions") + .expect("prompt"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(SelectiveRepairProvider::default()); + let runtime = Runtime::new(store, directory.path()) + .with_clock(Arc::new(FixedClock)) + .with_ids(Arc::new(SequenceIds::default())) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider)); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: prompt-repair } +spec: + providers: + fake: { kind: fake } + agents: + first: + provider: fake + model: fake + instructionsFile: first.txt + structuredOutput: + type: object + required: [value] + properties: + value: { type: string } + actions: + assert: { kind: builtin.assert } + tasks: + - id: first + uses: agent:first + with: { prompt: produce durable output } + - id: second + uses: action:assert + needs: [first] + with: { that: false } +"#; + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + std::fs::write(directory.path().join("first.txt"), "changed instructions") + .expect("changed prompt"); + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("plan"); + assert!(!plan.compatible); + assert!(plan.blocked_reuse.iter().any(|block| { + block.task_id == "first" && block.rule == "definition_fingerprint_mismatch" + })); + } + + #[tokio::test] + async fn repair_reconstructs_successful_memory_delta_and_excludes_failed_boundary() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: memory-repair } +spec: + memory: + working: { durable: false, failed: absent } + policy: { approval: never } + actions: + memory: { kind: builtin.memory.write } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:memory + with: { key: durable, value: true } + - id: second + uses: action:assert + needs: [first] + with: { that: false } + - id: verify + uses: action:assert + needs: [second] + with: { that: "${{ memory.durable }}" } +"#; + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + let outcome = runtime + .repair( + &repaired_workflow, + &repaired_plan, + plan, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair"); + assert_eq!(outcome.state, RunState::Succeeded); + let run = store.load_run(&outcome.run_id).expect("run"); + assert_eq!(run.working_memory["durable"], true); + assert_eq!(run.working_memory["failed"], "absent"); + } + + #[tokio::test] + async fn repair_blocks_when_reused_artifact_is_missing() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: artifact-repair } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:write + with: { path: artifact.txt, content: durable } + - id: second + uses: action:assert + needs: [first] + with: { that: false } +"#; + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let compatible = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("plan"); + assert!(compatible.compatible, "{:?}", compatible.blocked_reuse); + std::fs::remove_file(directory.path().join("artifact.txt")).expect("remove artifact"); + let runs_before = store.stats().expect("stats").runs; + assert!(matches!( + runtime + .repair( + &repaired_workflow, + &repaired_plan, + compatible, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await, + Err(RuntimeError::RepairBlocked { .. }) + )); + assert_eq!(store.stats().expect("stats").runs, runs_before); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("blocked plan"); + assert!(!plan.compatible); + assert!( + plan.blocked_reuse + .iter() + .any(|block| { block.task_id == "first" && block.rule == "artifact_integrity" }) + ); + } + + #[tokio::test] + async fn repair_blocks_tampered_reused_output_digest() { + let directory = tempdir().expect("tempdir"); + let database = directory.path().join("runtime.db"); + let store = SqliteStore::open(&database).expect("store"); + let runtime = runtime(store, directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: corrupt-output-repair } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:assign + with: { value: durable } + - id: second + uses: action:assert + needs: [first] + with: { that: false } +"#; + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + rusqlite::Connection::open(&database) + .expect("tamper connection") + .execute( + "UPDATE task_states SET output_json = ?3 WHERE run_id = ?1 AND task_id = ?2", + rusqlite::params![source_run_id, "first", r#"{"tampered":true}"#], + ) + .expect("tamper output"); + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("plan"); + assert!(!plan.compatible); + assert!( + plan.blocked_reuse.iter().any(|block| { + block.task_id == "first" && block.rule == "output_digest_mismatch" + }) + ); + } + + #[tokio::test] + async fn repair_handles_multiple_roots_new_descendants_removed_tasks_and_restart_acknowledgement() + { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: graph-repair } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - { id: prepare, uses: "action:assign", with: { value: stable } } + - { id: removed, uses: "action:assign", with: { value: obsolete } } + - { id: left, uses: "action:assert", needs: [prepare], with: { that: false } } + - { id: right, uses: "action:assert", needs: [prepare], with: { that: false } } + - { id: combine, uses: "action:assign", needs: [left, right], with: { value: combined } } +"#; + let target_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: graph-repair } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - { id: prepare, uses: "action:assign", with: { value: stable } } + - { id: left, uses: "action:assert", needs: [prepare], with: { that: true } } + - { id: right, uses: "action:assert", needs: [prepare], with: { that: true } } + - { id: combine, uses: "action:assign", needs: [left, right], with: { value: combined } } + - { id: verify, uses: "action:assert", needs: [combine], with: { that: true } } +"#; + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, task, .. }) => { + assert_eq!(task, "left"); + run_id + } + other => panic!("expected source failure, got {other:?}"), + }; + let (target_workflow, target_plan) = compile_fixture(target_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &target_workflow, + &target_plan, + &["left".to_owned(), "right".to_owned()], + false, + ) + .expect("multi-root plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + assert_eq!(plan.repair_roots, ["left", "right"]); + assert_eq!(plan.reused_tasks, ["prepare"]); + assert_eq!(plan.rerun_tasks, ["left", "right", "combine", "verify"]); + assert_eq!(plan.new_tasks, ["verify"]); + assert_eq!(plan.removed_tasks, ["removed"]); + let outcome = runtime + .repair( + &target_workflow, + &target_plan, + plan, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("multi-root repair"); + assert_eq!(outcome.state, RunState::Succeeded); + + let unrelated_yaml = target_yaml.replace( + " - { id: verify, uses: \"action:assert\", needs: [combine], with: { that: true } }\n", + concat!( + " - { id: verify, uses: \"action:assert\", needs: [combine], with: { that: true } }\n", + " - { id: unrelated, uses: \"action:assign\", with: { value: new } }\n" + ), + ); + let (unrelated_workflow, unrelated_plan) = compile_fixture(&unrelated_yaml); + let blocked = runtime + .plan_repair( + &source_run_id, + &unrelated_workflow, + &unrelated_plan, + &["left".to_owned(), "right".to_owned()], + false, + ) + .expect("blocked unrelated plan"); + assert!(!blocked.compatible); + assert!(blocked.blocked_reuse.iter().any(|block| { + block.task_id == "unrelated" && block.rule == "new_task_outside_repair_closure" + })); + + let successful_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: restart-successful } +spec: + actions: { assign: { kind: builtin.assign } } + tasks: [{ id: done, uses: "action:assign", with: { value: old } }] +"#; + let target_successful_yaml = successful_yaml.replace("value: old", "value: new"); + let (successful_workflow, successful_plan) = compile_fixture(successful_yaml); + let successful_run = runtime + .start( + &successful_workflow, + &successful_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("successful source"); + let (target_successful_workflow, target_successful_plan) = + compile_fixture(&target_successful_yaml); + let blocked = runtime + .plan_repair( + &successful_run.run_id, + &target_successful_workflow, + &target_successful_plan, + &["done".to_owned()], + false, + ) + .expect("restart plan"); + assert!(!blocked.compatible); + assert!(blocked.blocked_reuse.iter().any(|block| { + block.task_id == "done" && block.rule == "successful_root_requires_acknowledgement" + })); + let acknowledged = runtime + .plan_repair( + &successful_run.run_id, + &target_successful_workflow, + &target_successful_plan, + &["done".to_owned()], + true, + ) + .expect("acknowledged restart plan"); + assert!(acknowledged.compatible, "{:?}", acknowledged.blocked_reuse); + } + + #[tokio::test] + async fn repair_allows_confirmed_idempotent_effects_and_preserves_approval_gates() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: effect-safety } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + actions: + assert: { kind: builtin.assert } + write: { kind: builtin.write } + tasks: + - { id: publish, uses: "action:assert", with: { that: false } } +"#; + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let prior_effect = EffectRequest::new( + &source_run_id, + "publish", + 1, + 1, + "external.idempotent-put", + EffectClass::ExternalMutate, + Risk::Medium, + Idempotency::Idempotent, + serde_json::json!({"key": "stable", "value": 1}), + "put stable value", + "trace-idempotent", + ); + store + .record_effect_request(&prior_effect, FixedClock.now()) + .expect("record effect"); + store + .mark_effect_started(&prior_effect.id, FixedClock.now()) + .expect("start effect"); + store + .complete_effect( + &prior_effect.id, + Ok(&serde_json::json!({"stored": true})), + FixedClock.now(), + ) + .expect("complete effect"); + + let target_yaml = source_yaml.replace( + " - { id: publish, uses: \"action:assert\", with: { that: false } }", + " - { id: publish, uses: \"action:write\", with: { path: approved.txt, content: repaired } }", + ); + let (target_workflow, target_plan) = compile_fixture(&target_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &target_workflow, + &target_plan, + &["publish".to_owned()], + false, + ) + .expect("effect-safe plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + let outcome = runtime + .repair( + &target_workflow, + &target_plan, + plan, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("approval-paused repair"); + assert_eq!(outcome.state, RunState::Paused); + assert!(!directory.path().join("approved.txt").exists()); + let tasks = store.list_tasks(&outcome.run_id).expect("repair tasks"); + assert_eq!(tasks[0].state, TaskState::WaitingForApproval); + assert_eq!( + tasks[0].disposition, + agentctl_store::TaskDisposition::Executed + ); + assert_eq!( + store + .pending_approvals(&outcome.run_id) + .expect("approvals") + .len(), + 1 + ); + } + + #[tokio::test] + async fn deterministic_dataflow_condition_and_working_memory() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: dataflow } +spec: + inputs: { enabled: true, greeting: hello } + memory: + working: { count: 0 } + actions: + assign: { kind: builtin.assign } + remember: { kind: builtin.memory.write } + tasks: + - id: first + uses: action:assign + with: { message: "${{ inputs.greeting }}" } + - id: remember + uses: action:remember + needs: [first] + when: "${{ inputs.enabled == true }}" + with: { key: result, value: "${{ tasks.first.output.output.message }}" } +"#, + ); + let outcome = runtime(store.clone(), directory.path()) + .start( + &workflow, + &plan, + serde_json::json!({"enabled": true, "greeting": "hello"}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("run succeeds"); + assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!( + store.load_run(&outcome.run_id).expect("run").working_memory["result"], + "hello" + ); + } + + #[tokio::test] + async fn task_output_contract_failure_is_durable() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: output-contract } +spec: + actions: { assign: { kind: builtin.assign } } + tasks: + - id: typed + uses: action:assign + with: { value: not-an-integer } + outputSchema: + type: object + properties: { value: { type: integer } } + required: [value] +"#, + ); + let run_id = match runtime(store.clone(), directory.path()) + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { + run_id, message, .. + }) => { + assert!(message.contains("task output contract failed")); + run_id + } + other => panic!("expected contract failure, got {other:?}"), + }; + let task = &store.list_tasks(&run_id).expect("tasks")[0]; + assert_eq!(task.state, TaskState::Failed); + assert!( + task.error + .as_deref() + .is_some_and(|error| error.contains("task output contract failed")) ); } diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index e499b89..73de5f6 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -1,5 +1,6 @@ //! Versioned SQLite persistence for agentctl. +use std::collections::BTreeMap; use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -15,7 +16,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use thiserror::Error; -pub const DATABASE_SCHEMA_VERSION: u32 = 4; +pub const DATABASE_SCHEMA_VERSION: u32 = 5; pub const RUNTIME_STATE_VERSION: u32 = 1; pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; pub const AUDIT_EVENT_VERSION: u32 = 1; @@ -189,6 +190,31 @@ DROP TABLE tool_calls; ALTER TABLE tool_calls_v4 RENAME TO tool_calls; "#; +const MIGRATION_5: &str = r#" +ALTER TABLE runs ADD COLUMN source_run_id TEXT; +ALTER TABLE runs ADD COLUMN source_workflow_digest TEXT; +ALTER TABLE runs ADD COLUMN repair_roots_json TEXT; +ALTER TABLE runs ADD COLUMN repair_reason TEXT; +ALTER TABLE runs ADD COLUMN repair_format_version INTEGER; + +ALTER TABLE task_states ADD COLUMN disposition TEXT NOT NULL DEFAULT 'executed'; +ALTER TABLE task_states ADD COLUMN metadata_version INTEGER; +ALTER TABLE task_states ADD COLUMN source_run_id TEXT; +ALTER TABLE task_states ADD COLUMN source_task_id TEXT; +ALTER TABLE task_states ADD COLUMN source_attempt INTEGER; +ALTER TABLE task_states ADD COLUMN definition_fingerprint TEXT; +ALTER TABLE task_states ADD COLUMN input_digest TEXT; +ALTER TABLE task_states ADD COLUMN output_contract_fingerprint TEXT; +ALTER TABLE task_states ADD COLUMN output_digest TEXT; +ALTER TABLE task_states ADD COLUMN state_delta_json TEXT; +ALTER TABLE task_states ADD COLUMN state_delta_digest TEXT; +ALTER TABLE task_states ADD COLUMN artifact_manifest_json TEXT; +ALTER TABLE task_states ADD COLUMN reuse_decision_json TEXT; + +CREATE INDEX idx_runs_source_run ON runs(source_run_id); +CREATE INDEX idx_tasks_disposition ON task_states(run_id, disposition); +"#; + #[derive(Clone)] pub struct SqliteStore { connection: Arc>, @@ -235,6 +261,11 @@ pub struct RunRecord { pub state: RunState, pub mode: RunMode, pub parent_run_id: Option, + pub source_run_id: Option, + pub source_workflow_digest: Option, + pub repair_roots: Vec, + pub repair_reason: Option, + pub repair_format_version: Option, pub base_path: Option, pub cancellation_requested: bool, pub created_at: DateTime, @@ -248,6 +279,23 @@ pub enum RunMode { Check, Replay, Fork, + Repair, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskDisposition { + Executed, + Reused, + Recorded, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactRecord { + pub path: String, + pub digest: String, + pub size_bytes: u64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -260,9 +308,53 @@ pub struct TaskRecord { pub attempt: u16, pub output: Option, pub error: Option, + pub disposition: TaskDisposition, + pub metadata_version: Option, + pub source_run_id: Option, + pub source_task_id: Option, + pub source_attempt: Option, + pub definition_fingerprint: Option, + pub input_digest: Option, + pub output_contract_fingerprint: Option, + pub output_digest: Option, + pub state_delta: Option, + pub state_delta_digest: Option, + pub artifact_manifest: Vec, + pub reuse_decision: Option, pub updated_at: DateTime, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TaskExecutionMetadata { + pub metadata_version: u32, + pub definition_fingerprint: String, + pub input_digest: String, + pub output_contract_fingerprint: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TaskCompletionMetadata { + pub execution: TaskExecutionMetadata, + pub output_digest: String, + pub state_delta: Value, + pub state_delta_digest: String, + pub artifact_manifest: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReusedTaskMaterialization { + pub task_id: String, + pub source_run_id: String, + pub source_task_id: String, + pub source_attempt: u16, + pub output: Value, + pub metadata: TaskCompletionMetadata, + pub reuse_decision: Value, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ApprovalRequest { @@ -457,11 +549,139 @@ impl SqliteStore { Ok(()) } + #[allow(clippy::too_many_arguments)] + pub fn create_repair_run( + &self, + run_id: &str, + source_run_id: &str, + source_workflow_digest: &str, + workflow_schema_version: &str, + workflow: &Value, + plan: &CompiledPlan, + inputs: &Value, + working_memory: &Value, + repair_roots: &[String], + reason: Option<&str>, + reused_tasks: &[ReusedTaskMaterialization], + task_decisions: &Value, + base_path: &Path, + now: DateTime, + trace_id: &str, + ) -> Result<(), StoreError> { + let reused = reused_tasks + .iter() + .map(|task| (task.task_id.as_str(), task)) + .collect::>(); + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO runs (run_id, runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, state, mode, source_run_id, source_workflow_digest, repair_roots_json, repair_reason, repair_format_version, base_path, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, 1, ?17, ?18, ?18)", + params![ + run_id, + RUNTIME_STATE_VERSION, + plan.workflow_digest, + workflow_schema_version, + plan.plan_digest, + plan.format_version, + encode(workflow)?, + encode(plan)?, + encode(inputs)?, + encode(working_memory)?, + encode_enum(RunState::Running)?, + encode_enum(RunMode::Repair)?, + source_run_id, + source_workflow_digest, + encode(repair_roots)?, + reason, + base_path.display().to_string(), + now.to_rfc3339(), + ], + )?; + for (position, task_id) in plan.order.iter().enumerate() { + let position = i64::try_from(position).map_err(|_| { + StoreError::Incompatible("task position exceeds SQLite integer range".to_owned()) + })?; + if let Some(task) = reused.get(task_id.as_str()) { + transaction.execute( + "INSERT INTO task_states (run_id, task_id, position, state, attempt, output_json, disposition, metadata_version, source_run_id, source_task_id, source_attempt, definition_fingerprint, input_digest, output_contract_fingerprint, output_digest, state_delta_json, state_delta_digest, artifact_manifest_json, reuse_decision_json, updated_at) VALUES (?1, ?2, ?3, ?4, 0, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", + params![ + run_id, + task_id, + position, + encode_enum(TaskState::Succeeded)?, + encode(&task.output)?, + encode_enum(TaskDisposition::Reused)?, + task.metadata.execution.metadata_version, + task.source_run_id, + task.source_task_id, + task.source_attempt, + task.metadata.execution.definition_fingerprint, + task.metadata.execution.input_digest, + task.metadata.execution.output_contract_fingerprint, + task.metadata.output_digest, + encode(&task.metadata.state_delta)?, + task.metadata.state_delta_digest, + encode(&task.metadata.artifact_manifest)?, + encode(&task.reuse_decision)?, + now.to_rfc3339(), + ], + )?; + append_audit_tx( + &transaction, + run_id, + "repair.task_reused", + Some(task_id), + trace_id, + &serde_json::json!({ + "sourceRunId": task.source_run_id, + "sourceTaskId": task.source_task_id, + "sourceAttempt": task.source_attempt, + "outputDigest": task.metadata.output_digest, + "decision": task.reuse_decision, + }), + now, + )?; + } else { + transaction.execute( + "INSERT INTO task_states (run_id, task_id, position, state, disposition, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + run_id, + task_id, + position, + encode_enum(TaskState::Pending)?, + encode_enum(TaskDisposition::Executed)?, + now.to_rfc3339(), + ], + )?; + } + } + append_audit_tx( + &transaction, + run_id, + "repair.created", + None, + trace_id, + &serde_json::json!({ + "sourceRunId": source_run_id, + "sourceWorkflowDigest": source_workflow_digest, + "targetWorkflowDigest": plan.workflow_digest, + "repairRoots": repair_roots, + "reason": reason, + "reusedTasks": reused_tasks.iter().map(|task| task.task_id.as_str()).collect::>(), + "taskDecisions": task_decisions, + }), + now, + )?; + checkpoint_tx(&transaction, run_id, now)?; + transaction.commit()?; + Ok(()) + } + pub fn load_run(&self, run_id: &str) -> Result { let connection = self.connection.lock(); connection .query_row( - "SELECT runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, output_json, state, mode, parent_run_id, cancellation_requested, created_at, updated_at, base_path FROM runs WHERE run_id = ?1", + "SELECT runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, output_json, state, mode, parent_run_id, cancellation_requested, created_at, updated_at, base_path, source_run_id, source_workflow_digest, repair_roots_json, repair_reason, repair_format_version FROM runs WHERE run_id = ?1", [run_id], |row| { let state_version: u32 = row.get(0)?; @@ -484,6 +704,11 @@ impl SqliteStore { row.get::<_, String>(14)?, row.get::<_, String>(15)?, row.get::<_, Option>(16)?, + row.get::<_, Option>(17)?, + row.get::<_, Option>(18)?, + row.get::<_, Option>(19)?, + row.get::<_, Option>(20)?, + row.get::<_, Option>(21)?, )) }, ) @@ -515,6 +740,15 @@ impl SqliteStore { state: decode_enum(&row.10, "run.state")?, mode: decode_enum(&row.11, "run.mode")?, parent_run_id: row.12, + source_run_id: row.17, + source_workflow_digest: row.18, + repair_roots: row + .19 + .map(|value| decode(&value, "run.repair_roots")) + .transpose()? + .unwrap_or_default(), + repair_reason: row.20, + repair_format_version: row.21, base_path: row.16, cancellation_requested: row.13, created_at: parse_time(&row.14, "created_at")?, @@ -578,7 +812,7 @@ impl SqliteStore { pub fn list_tasks(&self, run_id: &str) -> Result, StoreError> { let connection = self.connection.lock(); let mut statement = connection.prepare( - "SELECT task_id, position, state, attempt, output_json, error, updated_at FROM task_states WHERE run_id = ?1 ORDER BY position", + "SELECT task_id, position, state, attempt, output_json, error, updated_at, disposition, metadata_version, source_run_id, source_task_id, source_attempt, definition_fingerprint, input_digest, output_contract_fingerprint, output_digest, state_delta_json, state_delta_digest, artifact_manifest_json, reuse_decision_json FROM task_states WHERE run_id = ?1 ORDER BY position", )?; let rows = statement.query_map([run_id], |row| { Ok(( @@ -589,6 +823,19 @@ impl SqliteStore { row.get::<_, Option>(4)?, row.get::<_, Option>(5)?, row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + row.get::<_, Option>(10)?, + row.get::<_, Option>(11)?, + row.get::<_, Option>(12)?, + row.get::<_, Option>(13)?, + row.get::<_, Option>(14)?, + row.get::<_, Option>(15)?, + row.get::<_, Option>(16)?, + row.get::<_, Option>(17)?, + row.get::<_, Option>(18)?, + row.get::<_, Option>(19)?, )) })?; rows.map(|row| { @@ -604,6 +851,29 @@ impl SqliteStore { .map(|value| decode(&value, "task.output")) .transpose()?, error: row.5, + disposition: decode_enum(&row.7, "task.disposition")?, + metadata_version: row.8, + source_run_id: row.9, + source_task_id: row.10, + source_attempt: row.11, + definition_fingerprint: row.12, + input_digest: row.13, + output_contract_fingerprint: row.14, + output_digest: row.15, + state_delta: row + .16 + .map(|value| decode(&value, "task.state_delta")) + .transpose()?, + state_delta_digest: row.17, + artifact_manifest: row + .18 + .map(|value| decode(&value, "task.artifact_manifest")) + .transpose()? + .unwrap_or_default(), + reuse_decision: row + .19 + .map(|value| decode(&value, "task.reuse_decision")) + .transpose()?, updated_at: parse_time(&row.6, "task.updated_at")?, }) }) @@ -668,6 +938,170 @@ impl SqliteStore { Ok(()) } + pub fn record_task_execution_metadata( + &self, + run_id: &str, + task_id: &str, + metadata: &TaskExecutionMetadata, + now: DateTime, + ) -> Result<(), StoreError> { + let changed = self.connection.lock().execute( + "UPDATE task_states SET metadata_version = ?3, definition_fingerprint = ?4, input_digest = ?5, output_contract_fingerprint = ?6, updated_at = ?7 WHERE run_id = ?1 AND task_id = ?2 AND state = ?8", + params![ + run_id, + task_id, + metadata.metadata_version, + metadata.definition_fingerprint, + metadata.input_digest, + metadata.output_contract_fingerprint, + now.to_rfc3339(), + encode_enum(TaskState::Running)?, + ], + )?; + if changed != 1 { + return Err(StoreError::InvalidTransition(format!( + "task `{task_id}` in run `{run_id}` is not running" + ))); + } + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub fn complete_task( + &self, + run_id: &str, + task_id: &str, + output: &Value, + working_memory: Option<&Value>, + metadata: &TaskCompletionMetadata, + now: DateTime, + trace_id: &str, + ) -> Result<(), StoreError> { + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current: String = transaction + .query_row( + "SELECT state FROM task_states WHERE run_id = ?1 AND task_id = ?2", + params![run_id, task_id], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| StoreError::TaskNotFound { + run_id: run_id.to_owned(), + task_id: task_id.to_owned(), + })?; + let current: TaskState = decode_enum(¤t, "task.state")?; + current + .transition(TaskState::Succeeded) + .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; + transaction.execute( + "UPDATE task_states SET state = ?3, output_json = ?4, error = NULL, disposition = ?5, metadata_version = ?6, definition_fingerprint = ?7, input_digest = ?8, output_contract_fingerprint = ?9, output_digest = ?10, state_delta_json = ?11, state_delta_digest = ?12, artifact_manifest_json = ?13, updated_at = ?14 WHERE run_id = ?1 AND task_id = ?2", + params![ + run_id, + task_id, + encode_enum(TaskState::Succeeded)?, + encode(output)?, + encode_enum(TaskDisposition::Executed)?, + metadata.execution.metadata_version, + metadata.execution.definition_fingerprint, + metadata.execution.input_digest, + metadata.execution.output_contract_fingerprint, + metadata.output_digest, + encode(&metadata.state_delta)?, + metadata.state_delta_digest, + encode(&metadata.artifact_manifest)?, + now.to_rfc3339(), + ], + )?; + if let Some(memory) = working_memory { + transaction.execute( + "UPDATE runs SET working_memory_json = ?2, updated_at = ?3 WHERE run_id = ?1", + params![run_id, encode(memory)?, now.to_rfc3339()], + )?; + } else { + transaction.execute( + "UPDATE runs SET updated_at = ?2 WHERE run_id = ?1", + params![run_id, now.to_rfc3339()], + )?; + } + append_audit_tx( + &transaction, + run_id, + "task.transition", + Some(task_id), + trace_id, + &serde_json::json!({ + "from": current, + "to": TaskState::Succeeded, + "disposition": TaskDisposition::Executed, + "outputDigest": metadata.output_digest, + "stateDeltaDigest": metadata.state_delta_digest, + }), + now, + )?; + checkpoint_tx(&transaction, run_id, now)?; + transaction.commit()?; + Ok(()) + } + + pub fn record_replayed_task_metadata( + &self, + replay_run_id: &str, + source: &TaskRecord, + now: DateTime, + trace_id: &str, + ) -> Result<(), StoreError> { + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let changed = transaction.execute( + "UPDATE task_states SET disposition = ?3, metadata_version = ?4, source_run_id = ?5, source_task_id = ?6, source_attempt = ?7, definition_fingerprint = ?8, input_digest = ?9, output_contract_fingerprint = ?10, output_digest = ?11, state_delta_json = ?12, state_delta_digest = ?13, artifact_manifest_json = ?14, reuse_decision_json = ?15, updated_at = ?16 WHERE run_id = ?1 AND task_id = ?2", + params![ + replay_run_id, + source.task_id, + encode_enum(TaskDisposition::Recorded)?, + source.metadata_version, + source.run_id, + source.task_id, + source.attempt, + source.definition_fingerprint, + source.input_digest, + source.output_contract_fingerprint, + source.output_digest, + source.state_delta.as_ref().map(encode).transpose()?, + source.state_delta_digest, + encode(&source.artifact_manifest)?, + encode(&serde_json::json!({ + "recordedFromRunId": source.run_id, + "sourceDisposition": source.disposition, + "sourceProvenance": source.reuse_decision, + }))?, + now.to_rfc3339(), + ], + )?; + if changed != 1 { + return Err(StoreError::TaskNotFound { + run_id: replay_run_id.to_owned(), + task_id: source.task_id.clone(), + }); + } + append_audit_tx( + &transaction, + replay_run_id, + "replay.task_recorded", + Some(&source.task_id), + trace_id, + &serde_json::json!({ + "sourceRunId": source.run_id, + "sourceTaskId": source.task_id, + "sourceDisposition": source.disposition, + "outputDigest": source.output_digest, + }), + now, + )?; + transaction.commit()?; + Ok(()) + } + pub fn update_run_state( &self, run_id: &str, @@ -898,6 +1332,62 @@ impl SqliteStore { .map_err(StoreError::from) } + pub fn reconcile_effect_not_applied( + &self, + effect_id: &str, + actor: &str, + reason: &str, + now: DateTime, + ) -> Result<(), StoreError> { + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let run_id: Option = transaction + .query_row( + "SELECT run_id FROM effects WHERE effect_id = ?1 AND status IN (?2, ?3)", + params![ + effect_id, + encode_enum(EffectStatus::Started)?, + encode_enum(EffectStatus::Uncertain)?, + ], + |row| row.get(0), + ) + .optional()?; + let Some(run_id) = run_id else { + return Err(StoreError::Incompatible(format!( + "effect `{effect_id}` is missing or is not uncertain" + ))); + }; + transaction.execute( + "UPDATE effects SET status = ?2, error = ?3, completed_at = ?4, confirmed = 0 WHERE effect_id = ?1", + params![ + effect_id, + encode_enum(EffectStatus::Failed)?, + format!("operator reconciled as not applied: {reason}"), + now.to_rfc3339(), + ], + )?; + transaction.execute( + "UPDATE tool_calls SET status = 'failed', completed_at = COALESCE(completed_at, ?2) WHERE run_id = ?1 AND effect_id = ?3 AND status IN ('started', 'uncertain')", + params![run_id, now.to_rfc3339(), effect_id], + )?; + append_audit_tx( + &transaction, + &run_id, + "effect.reconciled_not_applied", + None, + "operator-reconciliation", + &serde_json::json!({ + "effectId": effect_id, + "actor": actor, + "reason": reason, + "outcome": "not_applied", + }), + now, + )?; + transaction.commit()?; + Ok(()) + } + pub fn latest_effect_for_task( &self, run_id: &str, @@ -1485,6 +1975,7 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { (2_u32, MIGRATION_2), (3_u32, MIGRATION_3), (4_u32, MIGRATION_4), + (5_u32, MIGRATION_5), ]; for (version, sql) in migrations .into_iter() @@ -1636,6 +2127,17 @@ spec: .expect("create run"); } + fn create_version_four_database(path: &Path) { + let connection = Connection::open(path).expect("raw connection"); + connection.execute_batch(MIGRATION_1).expect("v1 schema"); + connection.execute_batch(MIGRATION_2).expect("v2 schema"); + connection.execute_batch(MIGRATION_3).expect("v3 schema"); + connection.execute_batch(MIGRATION_4).expect("v4 schema"); + connection + .pragma_update(None, "user_version", 4) + .expect("v4 marker"); + } + #[test] fn fresh_database_migrates_and_permissions_are_private() { let directory = tempdir().expect("temp dir"); @@ -1877,6 +2379,96 @@ spec: assert_eq!(store.stats().expect("stats").long_term_memory, 0); } + #[test] + fn upgrades_the_pre_repair_schema_and_creates_repair_records() { + let directory = tempdir().expect("temp dir"); + let path = directory.path().join("runtime.db"); + create_version_four_database(&path); + + let store = SqliteStore::open(&path).expect("upgrade"); + assert_eq!(store.schema_version(), DATABASE_SCHEMA_VERSION); + create(&store, "source"); + let (workflow, plan) = fixture(); + store + .create_repair_run( + "repair", + "source", + &plan.workflow_digest, + API_VERSION, + &workflow, + &plan, + &serde_json::json!({}), + &serde_json::json!({}), + &["one".to_owned()], + Some("migration test"), + &[], + &serde_json::json!([]), + Path::new("."), + Utc::now(), + "trace-repair", + ) + .expect("create repair"); + + let repair = store.load_run("repair").expect("repair run"); + assert_eq!(repair.mode, RunMode::Repair); + assert_eq!(repair.source_run_id.as_deref(), Some("source")); + assert_eq!(repair.repair_roots, ["one"]); + assert_eq!( + store.list_tasks("repair").expect("repair tasks")[0].disposition, + TaskDisposition::Executed + ); + } + + #[test] + fn interrupted_repair_migration_can_restart_cleanly() { + let directory = tempdir().expect("temp dir"); + let path = directory.path().join("runtime.db"); + create_version_four_database(&path); + let mut connection = Connection::open(&path).expect("raw connection"); + let transaction = connection.transaction().expect("migration transaction"); + transaction + .execute_batch("ALTER TABLE runs ADD COLUMN source_run_id TEXT;") + .expect("partial migration"); + transaction.rollback().expect("simulate interruption"); + drop(connection); + + let store = SqliteStore::open(&path).expect("restart migration"); + assert_eq!(store.schema_version(), DATABASE_SCHEMA_VERSION); + create(&store, "run"); + assert_eq!(store.list_tasks("run").expect("tasks").len(), 1); + } + + #[test] + fn repair_creation_rolls_back_every_row_on_materialization_failure() { + let store = SqliteStore::open_memory().expect("store"); + let (workflow, mut plan) = fixture(); + plan.order.push("one".to_owned()); + let result = store.create_repair_run( + "repair", + "source", + &plan.workflow_digest, + API_VERSION, + &workflow, + &plan, + &serde_json::json!({}), + &serde_json::json!({}), + &["one".to_owned()], + None, + &[], + &serde_json::json!([]), + Path::new("."), + Utc::now(), + "trace", + ); + + assert!(matches!(result, Err(StoreError::Sqlite(_)))); + assert!(matches!( + store.load_run("repair"), + Err(StoreError::RunNotFound(_)) + )); + assert_eq!(store.stats().expect("stats").runs, 0); + } + #[test] fn concurrent_readers_and_bounded_lock_wait_succeed() { let directory = tempdir().expect("temp dir"); diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 63fecb0..5745a36 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -71,6 +71,7 @@ dependencies = [ "async-trait", "chrono", "hex", + "jsonschema", "nix", "serde", "serde_json", diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index de827ef..dd5fe65 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -677,7 +677,8 @@ "failure": { "$ref": "#/$defs/FailureBehavior", "default": "stop" - } + }, + "outputSchema": true }, "additionalProperties": false, "required": [ From 3942aa0b72de3b6f6849a7faf292ab323100667e Mon Sep 17 00:00:00 2001 From: Ompragash Date: Thu, 23 Jul 2026 22:50:28 +0530 Subject: [PATCH 02/44] test: verify task-boundary repair and result reuse --- docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 49 + examples/selective-repair-openai/README.md | 26 + .../artifacts/.gitkeep | 1 + .../fixture/service.txt | 2 + .../repaired.workflow.yaml | 117 ++ .../source.workflow.yaml | 117 ++ xtask/src/acceptance.rs | 1001 ++++++++++++++++- xtask/src/main.rs | 138 ++- 8 files changed, 1443 insertions(+), 8 deletions(-) create mode 100644 docs/execution/EXAMPLE_VERIFICATION_MATRIX.md create mode 100644 examples/selective-repair-openai/README.md create mode 100644 examples/selective-repair-openai/artifacts/.gitkeep create mode 100644 examples/selective-repair-openai/fixture/service.txt create mode 100644 examples/selective-repair-openai/repaired.workflow.yaml create mode 100644 examples/selective-repair-openai/source.workflow.yaml diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md new file mode 100644 index 0000000..dbc110f --- /dev/null +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -0,0 +1,49 @@ +# Example verification matrix + +This inventory is enforced by `cargo xtask examples-verify`. The default command is credential-free: it discovers every YAML file under `examples/` and `fixtures/compat/`, requires one row per file, runs `check` and `plan` with the documented exit codes, and then runs the repository's canonical deterministic and mock journeys. `cargo xtask examples-verify-live-openai` is the explicit bounded live gate for every row marked OpenAI. + +`N/A` means the check does not apply to that example. `Canonical` means the behavior is exercised by the existing deterministic example or acceptance runner. `Opt-in` means credentials or an external service are required and the default gate does not claim execution. + +| Path | Purpose | Provider | Expected status | Check exit | Plan exit | Deterministic run | Mock run | Live run | Container run | Artifact verification | Output verification | Last deterministic result | +| --- | --- | --- | --- | ---: | ---: | --- | --- | --- | --- | --- | --- | --- | +| `examples/acceptance/mock-tool/workflow.yaml` | Full fake-provider tool journey | fake | success | 0 | 0 | Canonical | Canonical | N/A | Canonical | Canonical | Canonical | passed | +| `examples/custom-pack-tools/custom.pack.yaml` | Legacy custom pack manifest | N/A | pack manifest | 0 | 0 | N/A | N/A | N/A | N/A | Static | Static | inventoried | +| `examples/custom-pack-tools/mission.playbook.yaml` | Archived TypeScript custom-pack example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/dataflow/mission.playbook.yaml` | Archived TypeScript dataflow example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/demo.pack.yaml` | Legacy demonstration pack manifest | N/A | pack manifest | 0 | 0 | N/A | N/A | N/A | N/A | Static | Static | inventoried | +| `examples/docs/ci-quality-gate/workflow.yaml` | CI quality decision | deterministic | success and expected failure | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | +| `examples/docs/provider-portability/fake.yaml` | Portable fake-provider summary | fake | success | 0 | 0 | Canonical | Canonical | N/A | N/A | N/A | Canonical | passed | +| `examples/docs/provider-portability/openai.yaml` | Portable OpenAI summary | OpenAI | success | 0 | 0 | N/A | N/A | Passed 2026-07-23 | N/A | N/A | Live gate | live passed | +| `examples/docs/release-readiness/workflow.yaml` | Release evidence review | fake | success | 0 | 0 | Canonical | Canonical | N/A | N/A | N/A | Canonical | passed | +| `examples/docs/scheduled-review/workflow.yaml` | Scheduled deterministic review | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Canonical | Canonical | passed | +| `examples/hello.playbook.yaml` | Archived TypeScript hello example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/memory-flow/mission.playbook.yaml` | Archived TypeScript memory example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/openai-live/workflow.yaml` | Canonical OpenAI tool continuation | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | Blocked: Podman unavailable | Canonical | Canonical | live passed; container blocked | +| `examples/prompt-cache/mission.playbook.yaml` | Archived TypeScript prompt-cache example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/prompt-file-vars/mission.playbook.yaml` | Archived TypeScript prompt-file example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/real-autonomy/mission.playbook.yaml` | Archived TypeScript autonomy example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/remote-mcp-autonomy/mission.playbook.yaml` | Archived TypeScript MCP example | legacy | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON error | passed | +| `examples/selective-repair-openai/repaired.workflow.yaml` | Fixed two-agent repair target | OpenAI | success | 0 | 0 | Runtime mock | Runtime mock | Passed 2026-07-23 | Blocked: Podman unavailable | Marker artifact verified | Contract and marker | live passed; container blocked | +| `examples/selective-repair-openai/source.workflow.yaml` | Deliberately failed two-agent source | OpenAI | task 2 failure | 0 | 0 | Runtime mock | Runtime mock | Expected failure passed 2026-07-23 | Blocked: Podman unavailable | N/A | Durable task 1 output | live passed; container blocked | +| `examples/v1/a2a.yaml` | A2A delegation contract | A2A | external execution | 0 | 0 | Protocol mock | Protocol mock | N/A | N/A | N/A | Static | passed | +| `examples/v1/anthropic-live.yaml` | Anthropic native provider | Anthropic | credentialed execution | 0 | 0 | N/A | Protocol mock | External opt-in | N/A | N/A | Static | passed | +| `examples/v1/approval.yaml` | Approval-paused mutation | deterministic | paused | 0 | 0 | Acceptance equivalent | N/A | N/A | N/A | No write before approval | Canonical | passed | +| `examples/v1/capability-failure.yaml` | Negative capability contract | deterministic | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON diagnostics | passed | +| `examples/v1/check-diff.yaml` | Non-mutating check and diff | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | No mutation | Canonical | passed | +| `examples/v1/condition.yaml` | Conditional scheduling | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | +| `examples/v1/crash-resume.yaml` | Durable interruption fixture | fake | resumable | 0 | 0 | Acceptance equivalent | Canonical | N/A | N/A | N/A | Durable state | passed | +| `examples/v1/dataflow.yaml` | Typed task dataflow | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | +| `examples/v1/example.pack.yaml` | Native reusable pack manifest | N/A | pack manifest | 0 | 0 | Canonical consumer | N/A | N/A | N/A | Digest checked | Canonical | passed | +| `examples/v1/fake-provider.yaml` | Deterministic provider task | fake | success | 0 | 0 | Canonical | Canonical | N/A | N/A | N/A | Canonical | passed | +| `examples/v1/google-live.yaml` | Google native provider | Google | credentialed execution | 0 | 0 | N/A | Protocol mock | External opt-in | N/A | N/A | Static | passed | +| `examples/v1/hello.yaml` | Minimal assign workflow | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | +| `examples/v1/long-term-memory.yaml` | Namespaced durable memory | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | +| `examples/v1/mcp.yaml` | MCP call contract | MCP | external execution | 0 | 0 | Protocol mock | Protocol mock | N/A | N/A | N/A | Static | passed | +| `examples/v1/openai-live.yaml` | Minimal OpenAI response | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Live gate | live passed | +| `examples/v1/policy-denial.yaml` | Denied mutation | deterministic | policy failure | 0 | 0 | Canonical expected failure | N/A | N/A | N/A | No mutation | JSON error | passed | +| `examples/v1/reusable-pack.yaml` | Native reusable pack consumer | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Pack digest | Canonical | passed | +| `examples/v1/secret-reference.yaml` | Environment reference contract | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Secret-safe live gate | live passed | +| `examples/v1/working-memory.yaml` | Working-memory update | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | +| `fixtures/compat/v0/assign.playbook.yaml` | Language-neutral TypeScript compatibility fixture | legacy translator | success | 0 | 0 | Compatibility test | N/A | N/A | N/A | N/A | Oracle contract | passed | + +The live column is updated only by the opt-in command. Raw model content, databases, and credentials are never written to this committed matrix. diff --git a/examples/selective-repair-openai/README.md b/examples/selective-repair-openai/README.md new file mode 100644 index 0000000..48a088e --- /dev/null +++ b/examples/selective-repair-openai/README.md @@ -0,0 +1,26 @@ +# Selective repair with OpenAI + +This example is an opt-in live acceptance scenario for `agentctl repair`. + +The source workflow runs `analyze` successfully, then deliberately exhausts +`publish` after its first model-selected read-only tool call. The repaired +workflow changes only `publisher.maxTurns`. Repair from `publish` must reuse the +durable structured `analyze` output without another provider or tool dispatch, +execute deterministic verification, and write +`artifacts/selective-repair-result.txt`. + +```text +agentctl run source.workflow.yaml --workspace . --db repair.db +agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from publish --plan --workspace . --db repair.db +agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from publish --workspace . --db repair.db +agentctl inspect REPAIR_RUN_ID --db repair.db --output json +env -u OPENAI_API_KEY agentctl replay REPAIR_RUN_ID --db repair.db --output json +``` + +The first and third commands perform live OpenAI requests. Planning, inspection, +and recorded replay do not. `OPENAI_API_KEY` is read only from the environment. + +Run every credential-free example check with `cargo xtask examples-verify`. +The bounded all-OpenAI gate is +`cargo xtask examples-verify-live-openai`; it retains only sanitized metadata in +the ignored `.release-evidence/` directory. diff --git a/examples/selective-repair-openai/artifacts/.gitkeep b/examples/selective-repair-openai/artifacts/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/selective-repair-openai/artifacts/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/selective-repair-openai/fixture/service.txt b/examples/selective-repair-openai/fixture/service.txt new file mode 100644 index 0000000..8d40e7b --- /dev/null +++ b/examples/selective-repair-openai/fixture/service.txt @@ -0,0 +1,2 @@ +service=agentctl +marker=SELECTIVE_REPAIR_FIXTURE_CONFIRMED diff --git a/examples/selective-repair-openai/repaired.workflow.yaml b/examples/selective-repair-openai/repaired.workflow.yaml new file mode 100644 index 0000000..f3cecb3 --- /dev/null +++ b/examples/selective-repair-openai/repaired.workflow.yaml @@ -0,0 +1,117 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: selective-repair-openai +spec: + outputs: + upstream: "${{ tasks.analyze.output.marker }}" + repaired: "${{ tasks.publish.output.confirmed }}" + artifact: artifacts/selective-repair-result.txt + providers: + openai: + kind: openai + policy: + workspaceRoot: . + writableRoots: [artifacts] + networkAllowlist: [api.openai.com] + approval: never + tools: + read_fixture: + kind: builtin.workspace.read + description: Read the selective-repair UTF-8 fixture. + inputSchema: + type: object + properties: + path: { type: string } + required: [path] + additionalProperties: false + outputSchema: + type: object + properties: + path: { type: string } + content: { type: string } + bytes: { type: integer } + sha256: { type: string } + required: [path, content, bytes, sha256] + additionalProperties: false + capability: filesystem.read + risk: low + effectClass: observe + idempotency: idempotent + retrySafe: true + timeoutSeconds: 5 + approval: never + agents: + analyzer: + provider: openai + model: gpt-5.6 + instructions: | + Call read_fixture exactly once with {"path":"fixture/service.txt"}. + Return JSON matching the schema. Set marker to the value after "marker=" + and fixtureSha256 to the tool result's sha256. Do not invent values. + tools: [read_fixture] + maxTurns: 3 + maxToolCalls: 1 + maxOutputTokens: 128 + timeoutSeconds: 45 + reasoning: { effort: low } + structuredOutput: + type: object + properties: + marker: { type: string } + fixtureSha256: { type: string } + required: [marker, fixtureSha256] + additionalProperties: false + providerOptions: + store: true + reasoningContext: current_turn + parallelToolCalls: false + publisher: + provider: openai + model: gpt-5.6 + instructions: | + Call read_fixture exactly once with {"path":"fixture/service.txt"}. + After the tool result, return JSON matching the schema. Set confirmed to true + only when the fixture marker equals the upstreamMarker supplied in the prompt. + tools: [read_fixture] + maxTurns: 3 + maxToolCalls: 1 + maxOutputTokens: 128 + timeoutSeconds: 45 + reasoning: { effort: low } + structuredOutput: + type: object + properties: + confirmed: { type: boolean } + upstreamMarker: { type: string } + required: [confirmed, upstreamMarker] + additionalProperties: false + providerOptions: + store: true + reasoningContext: current_turn + parallelToolCalls: false + actions: + assert: { kind: builtin.assert } + write: { kind: builtin.write } + tasks: + - id: analyze + uses: agent:analyzer + with: + prompt: Analyze the fixture and produce the required durable structured output. + - id: publish + uses: agent:publisher + needs: [analyze] + with: + prompt: "Verify upstreamMarker=${{ tasks.analyze.output.marker }} against the fixture." + - id: verify + uses: action:assert + needs: [analyze, publish] + with: + that: "${{ tasks.publish.output.confirmed == true }}" + message: repaired publisher did not confirm the upstream marker + - id: artifact + uses: action:write + needs: [analyze, verify] + with: + path: artifacts/selective-repair-result.txt + content: "${{ tasks.analyze.output.marker }}" diff --git a/examples/selective-repair-openai/source.workflow.yaml b/examples/selective-repair-openai/source.workflow.yaml new file mode 100644 index 0000000..4664b93 --- /dev/null +++ b/examples/selective-repair-openai/source.workflow.yaml @@ -0,0 +1,117 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: selective-repair-openai +spec: + outputs: + upstream: "${{ tasks.analyze.output.marker }}" + repaired: "${{ tasks.publish.output.confirmed }}" + artifact: artifacts/selective-repair-result.txt + providers: + openai: + kind: openai + policy: + workspaceRoot: . + writableRoots: [artifacts] + networkAllowlist: [api.openai.com] + approval: never + tools: + read_fixture: + kind: builtin.workspace.read + description: Read the selective-repair UTF-8 fixture. + inputSchema: + type: object + properties: + path: { type: string } + required: [path] + additionalProperties: false + outputSchema: + type: object + properties: + path: { type: string } + content: { type: string } + bytes: { type: integer } + sha256: { type: string } + required: [path, content, bytes, sha256] + additionalProperties: false + capability: filesystem.read + risk: low + effectClass: observe + idempotency: idempotent + retrySafe: true + timeoutSeconds: 5 + approval: never + agents: + analyzer: + provider: openai + model: gpt-5.6 + instructions: | + Call read_fixture exactly once with {"path":"fixture/service.txt"}. + Return JSON matching the schema. Set marker to the value after "marker=" + and fixtureSha256 to the tool result's sha256. Do not invent values. + tools: [read_fixture] + maxTurns: 3 + maxToolCalls: 1 + maxOutputTokens: 128 + timeoutSeconds: 45 + reasoning: { effort: low } + structuredOutput: + type: object + properties: + marker: { type: string } + fixtureSha256: { type: string } + required: [marker, fixtureSha256] + additionalProperties: false + providerOptions: + store: true + reasoningContext: current_turn + parallelToolCalls: false + publisher: + provider: openai + model: gpt-5.6 + instructions: | + Call read_fixture exactly once with {"path":"fixture/service.txt"}. + After the tool result, return JSON matching the schema. Set confirmed to true + only when the fixture marker equals the upstreamMarker supplied in the prompt. + tools: [read_fixture] + maxTurns: 1 + maxToolCalls: 1 + maxOutputTokens: 128 + timeoutSeconds: 45 + reasoning: { effort: low } + structuredOutput: + type: object + properties: + confirmed: { type: boolean } + upstreamMarker: { type: string } + required: [confirmed, upstreamMarker] + additionalProperties: false + providerOptions: + store: true + reasoningContext: current_turn + parallelToolCalls: false + actions: + assert: { kind: builtin.assert } + write: { kind: builtin.write } + tasks: + - id: analyze + uses: agent:analyzer + with: + prompt: Analyze the fixture and produce the required durable structured output. + - id: publish + uses: agent:publisher + needs: [analyze] + with: + prompt: "Verify upstreamMarker=${{ tasks.analyze.output.marker }} against the fixture." + - id: verify + uses: action:assert + needs: [analyze, publish] + with: + that: "${{ tasks.publish.output.confirmed == true }}" + message: repaired publisher did not confirm the upstream marker + - id: artifact + uses: action:write + needs: [analyze, verify] + with: + path: artifacts/selective-repair-result.txt + content: "${{ tasks.analyze.output.marker }}" diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 4974d75..9b52cd0 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::env; use std::ffi::OsStr; use std::fs; @@ -8,11 +9,13 @@ use std::time::Duration; use anyhow::{Context, Result, bail, ensure}; use serde_json::Value; +use sha2::{Digest, Sha256}; use crate::process::{bounded_output, bounded_wait, configure_piped_command, output_diagnostics}; const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; +const ACCEPTANCE_SCENARIOS: usize = 28; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -708,7 +711,116 @@ pub fn run(root: &Path) -> Result<()> { )?; ensure_eq(&quickstart_run, "/data/output/verdict", VERIFY_TOKEN)?; - println!("agentctl credential-free acceptance passed (25 scenarios)"); + scenario( + 26, + "selective repair plans reuse, executes the suffix, and exposes lineage", + ); + let repair_source = workspace.join("repair-source.yaml"); + let repair_target = workspace.join("repair-target.yaml"); + write(&repair_source, SELECTIVE_REPAIR_SOURCE_WORKFLOW)?; + write(&repair_target, SELECTIVE_REPAIR_TARGET_WORKFLOW)?; + let repair_db = directory.path().join("repair.db"); + let source_failure = json_with_code( + &binary, + &workspace, + &run_args(&repair_source, &repair_db, &workspace, &[]), + 4, + )?; + let source_run_id = string_at(&source_failure, "/error/runId")?; + let repair_plan = successful_json( + &binary, + &workspace, + &strings([ + "repair", + path(&repair_target)?, + source_run_id, + "--from", + "second", + "--plan", + "--db", + path(&repair_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&repair_plan, "/data/compatible", true)?; + ensure_eq(&repair_plan, "/data/reusedTasks/0", "first")?; + ensure_eq(&repair_plan, "/data/rerunTasks/0", "second")?; + ensure_eq(&repair_plan, "/data/rerunTasks/1", "third")?; + let repair = successful_json( + &binary, + &workspace, + &strings([ + "repair", + path(&repair_target)?, + source_run_id, + "--from", + "second", + "--reason", + "acceptance fixture", + "--db", + path(&repair_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&repair, "/data/state", "succeeded")?; + ensure_eq(&repair, "/data/sourceRunId", source_run_id)?; + ensure_eq(&repair, "/data/reusedTasks/0", "first")?; + let repair_run_id = string_at(&repair, "/data/runId")?; + let repair_inspect = inspect(&binary, &workspace, &repair_db, repair_run_id)?; + ensure_eq(&repair_inspect, "/data/run/mode", "repair")?; + ensure_eq(&repair_inspect, "/data/run/sourceRunId", source_run_id)?; + ensure_eq(&repair_inspect, "/data/tasks/0/disposition", "reused")?; + ensure_eq(&repair_inspect, "/data/tasks/1/disposition", "executed")?; + ensure!(array_len(&repair_inspect, "/data/effects")? == 0); + + scenario( + 27, + "blocked repair plans are parseable and create no partial run", + ); + let incompatible = workspace.join("repair-incompatible.yaml"); + write( + &incompatible, + &SELECTIVE_REPAIR_TARGET_WORKFLOW.replace("value: durable", "value: changed"), + )?; + let blocked = json_with_code( + &binary, + &workspace, + &strings([ + "repair", + path(&incompatible)?, + source_run_id, + "--from", + "second", + "--plan", + "--db", + path(&repair_db)?, + "--output", + "json", + "--color", + "never", + ]), + 3, + )?; + ensure_eq(&blocked, "/data/compatible", false)?; + ensure_eq( + &blocked, + "/data/blockedReuse/0/rule", + "definition_fingerprint_mismatch", + )?; + + scenario( + 28, + "effect inspection and not-applied reconciliation unblock repair", + ); + uncertain_repair_acceptance(&binary, &workspace, directory.path())?; + + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } @@ -736,6 +848,82 @@ pub fn container(root: &Path) -> Result<()> { ensure!(array_len(&replay_inspect, "/data/effects")? == 0); ensure!(array_len(&replay_inspect, "/data/toolCalls")? == 0); + let repair_directory = tempfile::tempdir()?; + let repair_layout = container_layout(repair_directory.path(), false)?; + write( + &repair_layout.config.join("repair-source.yaml"), + SELECTIVE_REPAIR_SOURCE_WORKFLOW, + )?; + write( + &repair_layout.config.join("repair-target.yaml"), + SELECTIVE_REPAIR_TARGET_WORKFLOW, + )?; + let source_failure = container_agentctl( + &engine, + &repair_layout, + &[ + "run", + "/config/repair-source.yaml", + "--workspace", + "/workspace", + "--db", + "/state/repair.db", + "--output", + "json", + "--color", + "never", + ], + 4, + "OCI repair source", + )?; + let source_run_id = string_at(&source_failure, "/error/runId")?; + let repair_plan = container_agentctl( + &engine, + &repair_layout, + &[ + "repair", + "/config/repair-target.yaml", + source_run_id, + "--from", + "second", + "--plan", + "--workspace", + "/workspace", + "--db", + "/state/repair.db", + "--output", + "json", + "--color", + "never", + ], + 0, + "OCI repair plan", + )?; + ensure_eq(&repair_plan, "/data/reusedTasks/0", "first")?; + let repaired = container_agentctl( + &engine, + &repair_layout, + &[ + "repair", + "/config/repair-target.yaml", + source_run_id, + "--from", + "second", + "--workspace", + "/workspace", + "--db", + "/state/repair.db", + "--output", + "json", + "--color", + "never", + ], + 0, + "OCI selective repair", + )?; + ensure_eq(&repaired, "/data/state", "succeeded")?; + ensure_eq(&repaired, "/data/reusedTasks/0", "first")?; + let missing_directory = tempfile::tempdir()?; let missing = container_layout(missing_directory.path(), false)?; write(&missing.config.join("workflow.yaml"), OPENAI_AUTH_WORKFLOW)?; @@ -752,7 +940,7 @@ pub fn container(root: &Path) -> Result<()> { container_signal_acceptance(&engine, directory.path())?; println!( - "agentctl OCI acceptance passed: success, artifact, inspect, network-disabled replay, missing-secret, invalid-input, SIGTERM, non-root, read-only root, mounted state/artifacts" + "agentctl OCI acceptance passed: success, artifact, inspect, network-disabled replay, selective repair, missing-secret, invalid-input, SIGTERM, non-root, read-only root, mounted state/artifacts" ); Ok(()) } @@ -830,6 +1018,454 @@ pub fn live_openai(root: &Path) -> Result<()> { Ok(()) } +pub fn examples_live_openai(root: &Path) -> Result<()> { + ensure!( + env::var_os("OPENAI_API_KEY").is_some(), + "OPENAI_API_KEY is required for the explicit live example command" + ); + super::verify_example_matrix(root)?; + let expected_live = BTreeSet::from([ + "examples/docs/provider-portability/openai.yaml".to_owned(), + "examples/openai-live/workflow.yaml".to_owned(), + "examples/selective-repair-openai/repaired.workflow.yaml".to_owned(), + "examples/selective-repair-openai/source.workflow.yaml".to_owned(), + "examples/v1/openai-live.yaml".to_owned(), + "examples/v1/secret-reference.yaml".to_owned(), + ]); + let mut discovered_live = BTreeSet::new(); + collect_openai_workflows(&root.join("examples"), root, &mut discovered_live)?; + ensure!( + discovered_live == expected_live, + "live OpenAI inventory mismatch; discovered={discovered_live:?}, expected={expected_live:?}" + ); + super::package(root)?; + let binary = packaged_binary(root)?; + let mut requests = 0_usize; + let mut usage = UsageTotals::default(); + let mut tool_calls = 0_usize; + let mut example_runs = Vec::new(); + + for (example, expected_code) in [ + ("examples/openai-live/workflow.yaml", 0), + ("examples/v1/openai-live.yaml", 0), + ("examples/v1/secret-reference.yaml", 0), + ("examples/docs/provider-portability/openai.yaml", 0), + ] { + let directory = tempfile::tempdir()?; + let source = root.join(example); + let source_parent = source.parent().context("live example parent")?; + copy_directory(source_parent, directory.path())?; + let workflow = directory + .path() + .join(source.file_name().context("live example file name")?); + let db = directory.path().join("runtime.db"); + successful_json( + &binary, + directory.path(), + &strings([ + "auth", + "check", + path(&workflow)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + successful_json( + &binary, + directory.path(), + &strings([ + "plan", + path(&workflow)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let result = json_with_code( + &binary, + directory.path(), + &run_args(&workflow, &db, directory.path(), &[]), + expected_code, + )?; + let run_id = if expected_code == 0 { + string_at(&result, "/data/runId")? + } else { + string_at(&result, "/error/runId")? + }; + let evidence = inspect(&binary, directory.path(), &db, run_id)?; + assert_secret_absent(&evidence)?; + requests = requests.saturating_add(model_effects(&evidence)); + usage = usage.plus(usage_totals(&evidence)); + tool_calls = tool_calls.saturating_add(array_len(&evidence, "/data/toolCalls")?); + guard_live_budget(requests, &usage)?; + if example == "examples/openai-live/workflow.yaml" { + ensure_eq(&result, "/data/output/verdict", LIVE_VERIFY_TOKEN)?; + ensure!( + fs::read_to_string(directory.path().join("artifacts/openai-live-report.txt"))? + == LIVE_VERIFY_TOKEN + ); + } + example_runs.push(serde_json::json!({ + "example": example, + "model": "gpt-5.6", + "runId": run_id, + "status": evidence.pointer("/data/run/state"), + "requestCount": model_effects(&evidence), + "toolCallCount": array_len(&evidence, "/data/toolCalls")?, + })); + } + + let repair_directory = tempfile::tempdir()?; + let repair_workspace = repair_directory.path().join("selective-repair"); + copy_directory( + &root.join("examples/selective-repair-openai"), + &repair_workspace, + )?; + let source_workflow = repair_workspace.join("source.workflow.yaml"); + let target_workflow = repair_workspace.join("repaired.workflow.yaml"); + let repair_db = repair_workspace.join("runtime.db"); + successful_json( + &binary, + &repair_workspace, + &strings([ + "plan", + path(&source_workflow)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + successful_json( + &binary, + &repair_workspace, + &strings([ + "plan", + path(&target_workflow)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let source_result = json_with_code( + &binary, + &repair_workspace, + &run_args(&source_workflow, &repair_db, &repair_workspace, &[]), + 4, + )?; + let source_run_id = string_at(&source_result, "/error/runId")?; + let source_before = inspect(&binary, &repair_workspace, &repair_db, source_run_id)?; + ensure_eq(&source_before, "/data/run/state", "failed")?; + ensure_eq(&source_before, "/data/tasks/0/state", "succeeded")?; + ensure_eq(&source_before, "/data/tasks/1/state", "failed")?; + ensure_eq( + &source_before, + "/data/tasks/0/output/marker", + "SELECTIVE_REPAIR_FIXTURE_CONFIRMED", + )?; + ensure!(model_effects(&source_before) >= 3); + ensure!( + source_before + .pointer("/data/providerSessions/1/continuation") + .is_some(), + "failed source task did not retain provider continuation evidence" + ); + + let repair_plan = successful_json( + &binary, + &repair_workspace, + &strings([ + "repair", + path(&target_workflow)?, + source_run_id, + "--from", + "publish", + "--plan", + "--db", + path(&repair_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&repair_plan, "/data/compatible", true)?; + ensure_eq(&repair_plan, "/data/reusedTasks/0", "analyze")?; + ensure_eq(&repair_plan, "/data/rerunTasks/0", "publish")?; + let repair_result = successful_json( + &binary, + &repair_workspace, + &strings([ + "repair", + path(&target_workflow)?, + source_run_id, + "--from", + "publish", + "--reason", + "live selective-repair acceptance", + "--db", + path(&repair_db)?, + "--workspace", + path(&repair_workspace)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let repair_run_id = string_at(&repair_result, "/data/runId")?; + let repair_evidence = inspect(&binary, &repair_workspace, &repair_db, repair_run_id)?; + ensure_eq(&repair_evidence, "/data/run/mode", "repair")?; + ensure_eq(&repair_evidence, "/data/run/sourceRunId", source_run_id)?; + ensure_eq(&repair_evidence, "/data/tasks/0/disposition", "reused")?; + ensure_eq(&repair_evidence, "/data/tasks/1/disposition", "executed")?; + ensure_eq( + &repair_result, + "/data/output/upstream", + "SELECTIVE_REPAIR_FIXTURE_CONFIRMED", + )?; + ensure_eq(&repair_result, "/data/output/repaired", true)?; + ensure!( + count_task_items(&repair_evidence, "/data/effects", "analyze", true) == 0, + "reused analyze task emitted a fresh effect" + ); + ensure!( + count_task_items(&repair_evidence, "/data/providerSessions", "analyze", false) == 0, + "reused analyze task created a provider session" + ); + ensure!( + count_task_items(&repair_evidence, "/data/toolCalls", "analyze", false) == 0, + "reused analyze task called a tool" + ); + ensure!( + count_task_items(&repair_evidence, "/data/providerSessions", "publish", false) == 1, + "repaired publish task did not use one fresh task-local provider session" + ); + ensure!( + source_before.pointer("/data/providerSessions/1/continuation") + != repair_evidence.pointer("/data/providerSessions/0/continuation"), + "repair continued the failed source provider session" + ); + ensure!( + count_task_items(&repair_evidence, "/data/toolCalls", "publish", false) == 1, + "repaired publish task did not execute the model-selected tool" + ); + let artifact = repair_workspace.join("artifacts/selective-repair-result.txt"); + ensure!( + fs::read_to_string(&artifact)? == "SELECTIVE_REPAIR_FIXTURE_CONFIRMED", + "selective repair artifact did not contain the reused upstream marker" + ); + let artifact_digest_before = hex::encode(Sha256::digest(fs::read(&artifact)?)); + + let source_after = inspect(&binary, &repair_workspace, &repair_db, source_run_id)?; + ensure!( + source_before.pointer("/data/run") == source_after.pointer("/data/run") + && source_before.pointer("/data/tasks") == source_after.pointer("/data/tasks"), + "repair mutated the terminal source run" + ); + let replay_result = json_with_removed_env( + &binary, + &repair_workspace, + &strings([ + "replay", + repair_run_id, + "--db", + path(&repair_db)?, + "--output", + "json", + "--color", + "never", + ]), + "OPENAI_API_KEY", + 0, + )?; + let replay_run_id = string_at(&replay_result, "/data/runId")?; + let replay_evidence = inspect(&binary, &repair_workspace, &repair_db, replay_run_id)?; + ensure!(array_len(&replay_evidence, "/data/effects")? == 0); + ensure!(array_len(&replay_evidence, "/data/toolCalls")? == 0); + ensure!(array_len(&replay_evidence, "/data/providerSessions")? == 0); + ensure!(replay_result.pointer("/data/output") == repair_result.pointer("/data/output")); + ensure!( + artifact_digest_before == hex::encode(Sha256::digest(fs::read(&artifact)?)), + "offline replay changed the repair artifact" + ); + + for evidence in [&source_before, &repair_evidence] { + assert_secret_absent(evidence)?; + requests = requests.saturating_add(model_effects(evidence)); + usage = usage.plus(usage_totals(evidence)); + tool_calls = tool_calls.saturating_add(array_len(evidence, "/data/toolCalls")?); + } + guard_live_budget(requests, &usage)?; + example_runs.push(serde_json::json!({ + "example": "examples/selective-repair-openai/source.workflow.yaml", + "model": "gpt-5.6", + "runId": source_run_id, + "status": "failed", + "requestCount": model_effects(&source_before), + "toolCallCount": array_len(&source_before, "/data/toolCalls")?, + })); + example_runs.push(serde_json::json!({ + "example": "examples/selective-repair-openai/repaired.workflow.yaml", + "model": "gpt-5.6", + "runId": repair_run_id, + "status": "succeeded", + "requestCount": model_effects(&repair_evidence), + "toolCallCount": array_len(&repair_evidence, "/data/toolCalls")?, + "reusedUpstream": true, + "replayRunId": replay_run_id, + "replayFreshEffects": 0, + })); + + write_live_summary( + root, + &example_runs, + requests, + tool_calls, + &usage, + source_run_id, + repair_run_id, + replay_run_id, + "local-complete-container-pending", + )?; + + let engine = container_engine()?; + ensure_engine_ready(&engine)?; + build_image(root, &engine)?; + let container_directory = tempfile::tempdir()?; + let layout = container_layout(container_directory.path(), true)?; + let source_container = SELECTIVE_REPAIR_SOURCE_WORKFLOW + .replace("writableRoots: [artifacts]", "writableRoots: [/artifacts]") + .replace( + "artifacts/selective-repair-result.txt", + "/artifacts/selective-repair-result.txt", + ); + let target_container = SELECTIVE_REPAIR_TARGET_WORKFLOW + .replace("writableRoots: [artifacts]", "writableRoots: [/artifacts]") + .replace( + "artifacts/selective-repair-result.txt", + "/artifacts/selective-repair-result.txt", + ); + write( + &layout.workspace.join("fixture/service.txt"), + "service=agentctl\nmarker=SELECTIVE_REPAIR_FIXTURE_CONFIRMED\n", + )?; + write(&layout.config.join("repair-source.yaml"), &source_container)?; + write(&layout.config.join("repair-target.yaml"), &target_container)?; + let container_source_result = container_agentctl_with_openai( + &engine, + &layout, + &[ + "run", + "/config/repair-source.yaml", + "--workspace", + "/workspace", + "--db", + "/state/runtime.db", + "--output", + "json", + "--color", + "never", + ], + 4, + "live OCI repair source", + )?; + let container_source_id = string_at(&container_source_result, "/error/runId")?; + let container_plan = container_agentctl( + &engine, + &layout, + &[ + "repair", + "/config/repair-target.yaml", + container_source_id, + "--from", + "publish", + "--plan", + "--workspace", + "/workspace", + "--db", + "/state/runtime.db", + "--output", + "json", + "--color", + "never", + ], + 0, + "live OCI repair plan", + )?; + ensure_eq(&container_plan, "/data/reusedTasks/0", "analyze")?; + let container_repair_result = container_agentctl_with_openai( + &engine, + &layout, + &[ + "repair", + "/config/repair-target.yaml", + container_source_id, + "--from", + "publish", + "--workspace", + "/workspace", + "--db", + "/state/runtime.db", + "--output", + "json", + "--color", + "never", + ], + 0, + "live OCI selective repair", + )?; + let container_repair_id = string_at(&container_repair_result, "/data/runId")?; + let container_source_evidence = inspect_container(&engine, &layout, container_source_id)?; + let container_repair_evidence = inspect_container(&engine, &layout, container_repair_id)?; + ensure!(count_task_items(&container_repair_evidence, "/data/effects", "analyze", true) == 0); + ensure!( + count_task_items( + &container_repair_evidence, + "/data/toolCalls", + "analyze", + false + ) == 0 + ); + ensure!( + fs::read_to_string(layout.artifacts.join("selective-repair-result.txt"))? + == "SELECTIVE_REPAIR_FIXTURE_CONFIRMED" + ); + let container_replay = replay_container(&engine, &layout, container_repair_id)?; + let container_replay_id = string_at(&container_replay, "/data/runId")?; + let container_replay_evidence = inspect_container(&engine, &layout, container_replay_id)?; + ensure!(array_len(&container_replay_evidence, "/data/effects")? == 0); + ensure!(array_len(&container_replay_evidence, "/data/toolCalls")? == 0); + ensure!(array_len(&container_replay_evidence, "/data/providerSessions")? == 0); + for evidence in [&container_source_evidence, &container_repair_evidence] { + assert_secret_absent(evidence)?; + requests = requests.saturating_add(model_effects(evidence)); + usage = usage.plus(usage_totals(evidence)); + tool_calls = tool_calls.saturating_add(array_len(evidence, "/data/toolCalls")?); + } + guard_live_budget(requests, &usage)?; + write_live_summary( + root, + &example_runs, + requests, + tool_calls, + &usage, + source_run_id, + repair_run_id, + replay_run_id, + "local-and-container-complete", + )?; + println!( + "live OpenAI example verification passed: examples=6 model=gpt-5.6 requests={requests} inputTokens={} outputTokens={} reasoningTokens={} cacheReadTokens={} cacheWriteTokens={} toolCalls={tool_calls} sourceRunId={source_run_id} repairRunId={repair_run_id} replayRunId={replay_run_id} containerSourceRunId={container_source_id} containerRepairRunId={container_repair_id} containerReplayRunId={container_replay_id}", + usage.input, usage.output, usage.reasoning, usage.cache_read, usage.cache_write, + ); + Ok(()) +} + fn run_error( binary: &Path, cwd: &Path, @@ -885,6 +1521,108 @@ fn signal_acceptance(binary: &Path, workspace: &Path, directory: &Path) -> Resul Ok(()) } +fn uncertain_repair_acceptance(binary: &Path, workspace: &Path, directory: &Path) -> Result<()> { + #[cfg(unix)] + { + let source = workspace.join("repair-uncertain-source.yaml"); + let target = workspace.join("repair-uncertain-target.yaml"); + write(&source, UNCERTAIN_REPAIR_SOURCE_WORKFLOW)?; + write(&target, UNCERTAIN_REPAIR_TARGET_WORKFLOW)?; + let db = directory.join("repair-uncertain.db"); + let failure = json_with_code( + binary, + workspace, + &run_args(&source, &db, workspace, &[]), + 4, + )?; + let run_id = string_at(&failure, "/error/runId")?; + let effects = successful_json( + binary, + workspace, + &strings([ + "effects", + "--db", + path(&db)?, + "inspect", + run_id, + "--task", + "work", + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&effects, "/data/effects/0/status", "uncertain")?; + let effect_id = string_at(&effects, "/data/effects/0/request/id")?; + let blocked = json_with_code( + binary, + workspace, + &strings([ + "repair", + path(&target)?, + run_id, + "--from", + "work", + "--plan", + "--db", + path(&db)?, + "--output", + "json", + "--color", + "never", + ]), + 3, + )?; + ensure_eq(&blocked, "/data/blockedReuse/0/rule", "unreconciled_effect")?; + successful_json( + binary, + workspace, + &strings([ + "effects", + "--db", + path(&db)?, + "reconcile", + effect_id, + "--outcome", + "not-applied", + "--actor", + "acceptance", + "--reason", + "subprocess was terminated before external mutation", + "--output", + "json", + "--color", + "never", + ]), + )?; + let repaired = successful_json( + binary, + workspace, + &strings([ + "repair", + path(&target)?, + run_id, + "--from", + "work", + "--db", + path(&db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&repaired, "/data/state", "succeeded")?; + } + #[cfg(not(unix))] + { + let _ = (binary, workspace, directory); + println!("uncertain subprocess repair acceptance is not applicable on this platform"); + } + Ok(()) +} + fn read_only_write_acceptance(binary: &Path, directory: &Path) -> Result<()> { #[cfg(unix)] { @@ -995,6 +1733,7 @@ struct UsageTotals { reasoning: u64, cache_read: u64, cache_write: u64, + cost_microusd: u64, } impl UsageTotals { @@ -1005,18 +1744,22 @@ impl UsageTotals { reasoning: self.reasoning.saturating_add(other.reasoning), cache_read: self.cache_read.saturating_add(other.cache_read), cache_write: self.cache_write.saturating_add(other.cache_write), + cost_microusd: self.cost_microusd.saturating_add(other.cost_microusd), } } } fn usage_totals(value: &Value) -> UsageTotals { let mut total = UsageTotals::default(); - let Some(tasks) = value.pointer("/data/tasks").and_then(Value::as_array) else { + let Some(effects) = value.pointer("/data/effects").and_then(Value::as_array) else { return total; }; - for usage in tasks + for usage in effects .iter() - .filter_map(|task| task.pointer("/output/usage")) + .filter(|effect| { + effect.pointer("/request/effectClass") == Some(&Value::String("model".to_owned())) + }) + .filter_map(|effect| effect.pointer("/result/usage")) { total.input = total .input @@ -1033,10 +1776,154 @@ fn usage_totals(value: &Value) -> UsageTotals { total.cache_write = total .cache_write .saturating_add(usage["cacheWriteTokens"].as_u64().unwrap_or(0)); + total.cost_microusd = total + .cost_microusd + .saturating_add(usage["costMicrousd"].as_u64().unwrap_or(0)); } total } +fn guard_live_budget(requests: usize, usage: &UsageTotals) -> Result<()> { + const MAX_REQUESTS: usize = 40; + const MAX_COST_MICROUSD: u64 = 10_000_000; + const CONSERVATIVE_INPUT_MICROUSD_PER_MILLION: u64 = 10_000_000; + const CONSERVATIVE_OUTPUT_MICROUSD_PER_MILLION: u64 = 50_000_000; + + ensure!( + requests <= MAX_REQUESTS, + "live request budget exceeded: {requests} > {MAX_REQUESTS}" + ); + let conservative_cost = usage + .input + .saturating_mul(CONSERVATIVE_INPUT_MICROUSD_PER_MILLION) + .saturating_div(1_000_000) + .saturating_add( + usage + .output + .saturating_add(usage.reasoning) + .saturating_mul(CONSERVATIVE_OUTPUT_MICROUSD_PER_MILLION) + .saturating_div(1_000_000), + ); + let guarded_cost = if usage.cost_microusd == 0 { + conservative_cost + } else { + usage.cost_microusd.max(conservative_cost) + }; + ensure!( + guarded_cost < MAX_COST_MICROUSD, + "live cost guard reached USD 10" + ); + Ok(()) +} + +fn count_task_items(value: &Value, pointer: &str, task_id: &str, nested_request: bool) -> usize { + value + .pointer(pointer) + .and_then(Value::as_array) + .map_or(0, |items| { + items + .iter() + .filter(|item| { + let pointer = if nested_request { + "/request/taskId" + } else { + "/taskId" + }; + item.pointer(pointer).and_then(Value::as_str) == Some(task_id) + }) + .count() + }) +} + +fn copy_directory(source: &Path, destination: &Path) -> Result<()> { + fs::create_dir_all(destination)?; + for entry in fs::read_dir(source)? { + let entry = entry?; + let source_path = entry.path(); + let destination_path = destination.join(entry.file_name()); + if source_path.is_dir() { + copy_directory(&source_path, &destination_path)?; + } else { + fs::copy(&source_path, &destination_path)?; + } + } + Ok(()) +} + +fn collect_openai_workflows( + directory: &Path, + root: &Path, + output: &mut BTreeSet, +) -> Result<()> { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_openai_workflows(&path, root, output)?; + } else if matches!( + path.extension().and_then(OsStr::to_str), + Some("yaml" | "yml") + ) { + let source = fs::read_to_string(&path)?; + if source.contains("apiVersion: agentctl.dev/v1alpha1") + && source.contains("kind: openai") + { + output.insert( + path.strip_prefix(root) + .context("OpenAI example outside repository")? + .to_string_lossy() + .replace('\\', "/"), + ); + } + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn write_live_summary( + root: &Path, + examples: &[Value], + requests: usize, + tool_calls: usize, + usage: &UsageTotals, + source_run_id: &str, + repair_run_id: &str, + replay_run_id: &str, + status: &str, +) -> Result<()> { + let directory = root.join(".release-evidence/selective-repair"); + fs::create_dir_all(&directory)?; + let value = serde_json::json!({ + "formatVersion": 1, + "status": status, + "model": "gpt-5.6", + "requestCount": requests, + "toolCallCount": tool_calls, + "usage": { + "inputTokens": usage.input, + "outputTokens": usage.output, + "reasoningTokens": usage.reasoning, + "cacheReadTokens": usage.cache_read, + "cacheWriteTokens": usage.cache_write, + "providerReportedCostMicrousd": usage.cost_microusd, + }, + "selectiveRepair": { + "sourceRunId": source_run_id, + "repairRunId": repair_run_id, + "replayRunId": replay_run_id, + "upstreamReused": true, + "replayFreshEffects": 0, + }, + "examples": examples, + }); + fs::write( + directory.join("live-summary.json"), + format!("{}\n", serde_json::to_string_pretty(&value)?), + )?; + Ok(()) +} + fn assert_error_metadata(value: &Value) -> Result<()> { ensure!( value @@ -1204,7 +2091,7 @@ fn command(root: &Path, program: &str, args: &[&str]) -> Result<()> { } fn scenario(number: usize, label: &str) { - println!("[{number}/25] {label}"); + println!("[{number}/{ACCEPTANCE_SCENARIOS}] {label}"); } fn write(path: &Path, contents: &str) -> Result<()> { @@ -1572,6 +2459,32 @@ fn replay_container(engine: &Path, layout: &ContainerLayout, run_id: &str) -> Re parse_output(&output_with_code(command, 0, "keyless OCI replay")?) } +fn container_agentctl( + engine: &Path, + layout: &ContainerLayout, + args: &[&str], + code: i32, + label: &str, +) -> Result { + let mut command = container_base(engine, layout)?; + command.args(["--network", "none", "agentctl-acceptance:local"]); + command.args(args); + parse_output(&output_with_code(command, code, label)?) +} + +fn container_agentctl_with_openai( + engine: &Path, + layout: &ContainerLayout, + args: &[&str], + code: i32, + label: &str, +) -> Result { + let mut command = container_base(engine, layout)?; + command.args(["--env", "OPENAI_API_KEY", "agentctl-acceptance:local"]); + command.args(args); + parse_output(&output_with_code(command, code, label)?) +} + fn container_signal_acceptance(engine: &Path, root: &Path) -> Result<()> { let layout = container_layout(&root.join("container-signal"), false)?; write( @@ -1771,6 +2684,82 @@ spec: with: { path: ../escaped.txt, content: blocked } "#; +const SELECTIVE_REPAIR_SOURCE_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: selective-repair-acceptance } +spec: + outputs: + repaired: "${{ tasks.third.output.output.value }}" + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:assign + with: { value: durable } + - id: second + uses: action:assert + needs: [first] + with: { that: false, message: deliberately broken } + - id: third + uses: action:assign + needs: [second] + with: { value: repaired } +"#; + +const SELECTIVE_REPAIR_TARGET_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: selective-repair-acceptance } +spec: + outputs: + repaired: "${{ tasks.third.output.output.value }}" + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:assign + with: { value: durable } + - id: second + uses: action:assert + needs: [first] + with: { that: true, message: fixed } + - id: third + uses: action:assign + needs: [second] + with: { value: repaired } +"#; + +const UNCERTAIN_REPAIR_SOURCE_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: uncertain-repair-acceptance } +spec: + policy: + processAllowlist: [sh] + approval: never + actions: + wait: + kind: builtin.shell.exec + command: /bin/sh + args: [-c, "sleep 2"] + timeoutSeconds: 1 + tasks: + - { id: work, uses: "action:wait" } +"#; + +const UNCERTAIN_REPAIR_TARGET_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: uncertain-repair-acceptance } +spec: + policy: + processAllowlist: [sh] + approval: never + actions: + assign: { kind: builtin.assign } + tasks: + - { id: work, uses: "action:assign", with: { recovered: true } } +"#; + const CONTAINER_MOCK_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 kind: Workflow metadata: { name: container-mock } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ded4f26..2545c81 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,3 +1,4 @@ +use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::ffi::OsStr; use std::fmt::Write as _; @@ -26,6 +27,8 @@ fn main() -> Result<()> { "acceptance" => acceptance::run(&root), "acceptance-container" => acceptance::container(&root), "acceptance-live-openai" => acceptance::live_openai(&root), + "examples-verify" => examples_verify(&root), + "examples-verify-live-openai" => acceptance::examples_live_openai(&root), "generate" => generate(&root), "package" => package(&root), "secret-scan" => { @@ -34,7 +37,7 @@ fn main() -> Result<()> { } "help" | "--help" | "-h" => { println!( - "cargo xtask verify\ncargo xtask docs-verify\ncargo xtask acceptance\ncargo xtask acceptance-container\ncargo xtask acceptance-live-openai\ncargo xtask generate\ncargo xtask package\ncargo xtask secret-scan" + "cargo xtask verify\ncargo xtask docs-verify\ncargo xtask acceptance\ncargo xtask acceptance-container\ncargo xtask acceptance-live-openai\ncargo xtask examples-verify\ncargo xtask examples-verify-live-openai\ncargo xtask generate\ncargo xtask package\ncargo xtask secret-scan" ); Ok(()) } @@ -198,8 +201,9 @@ fn verify(root: &Path) -> Result<()> { println!("[6/12] generated schema and CLI reference consistency"); verify_generated(root)?; - println!("[7/12] examples and negative contracts"); + println!("[7/12] examples, inventory, and negative contracts"); verify_examples(root)?; + verify_example_matrix(root)?; println!("[8/12] dependency sources and license metadata"); verify_metadata(root)?; @@ -221,6 +225,16 @@ fn verify(root: &Path) -> Result<()> { Ok(()) } +fn examples_verify(root: &Path) -> Result<()> { + run(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; + verify_example_matrix(root)?; + verify_examples(root)?; + verify_docs_examples(root)?; + verify_markdown_links(root)?; + println!("agentctl credential-free example verification passed"); + Ok(()) +} + fn generate(root: &Path) -> Result<()> { run(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; let binary = binary_path(root); @@ -275,8 +289,12 @@ fn generated_cli_reference(binary: &Path) -> Result { &["resume"], &["replay"], &["fork"], + &["repair"], &["cancel"], &["inspect"], + &["effects"], + &["effects", "inspect"], + &["effects", "reconcile"], &["approvals"], &["approvals", "list"], &["approvals", "approve"], @@ -418,6 +436,119 @@ fn verify_examples(root: &Path) -> Result<()> { Ok(()) } +#[derive(Debug)] +struct ExampleMatrixRow { + path: String, + check_code: i32, + plan_code: i32, +} + +fn verify_example_matrix(root: &Path) -> Result<()> { + let matrix_path = root.join("docs/execution/EXAMPLE_VERIFICATION_MATRIX.md"); + let matrix = fs::read_to_string(&matrix_path) + .with_context(|| format!("read {}", matrix_path.display()))?; + let mut rows = BTreeMap::new(); + for line in matrix.lines().filter(|line| line.starts_with("| `")) { + let columns = line.split('|').skip(1).map(str::trim).collect::>(); + if columns.len() < 13 { + bail!( + "{} has an incomplete example row: {line}", + matrix_path.display() + ); + } + let path = columns[0] + .strip_prefix('`') + .and_then(|value| value.strip_suffix('`')) + .context("matrix path must be wrapped in backticks")? + .to_owned(); + let check_code = columns[4] + .parse::() + .with_context(|| format!("matrix check code for {path}"))?; + let plan_code = columns[5] + .parse::() + .with_context(|| format!("matrix plan code for {path}"))?; + if rows + .insert( + path.clone(), + ExampleMatrixRow { + path, + check_code, + plan_code, + }, + ) + .is_some() + { + bail!("duplicate example matrix row"); + } + } + + let mut discovered = Vec::new(); + collect_yaml_files(&root.join("examples"), root, &mut discovered)?; + collect_yaml_files(&root.join("fixtures/compat"), root, &mut discovered)?; + let discovered = discovered.into_iter().collect::>(); + let documented = rows.keys().cloned().collect::>(); + if discovered != documented { + let missing = discovered.difference(&documented).collect::>(); + let stale = documented.difference(&discovered).collect::>(); + bail!("example matrix inventory mismatch; missing={missing:?}, stale={stale:?}"); + } + + let binary = binary_path(root); + for row in rows.values() { + if row.path.ends_with(".pack.yaml") { + continue; + } + let workflow = root.join(&row.path); + for (command_name, expected_code) in [("check", row.check_code), ("plan", row.plan_code)] { + let mut command = Command::new(&binary); + command + .current_dir(workflow.parent().context("workflow parent")?) + .arg(command_name) + .arg(&workflow) + .args(["--output", "json", "--color", "never"]); + let output = bounded_output(command, "example matrix validation") + .with_context(|| format!("{command_name} {}", row.path))?; + if output.status.code() != Some(expected_code) { + bail!( + "{command_name} {} returned {:?}, expected {expected_code}\n{}", + row.path, + output.status.code(), + output_diagnostics(&output) + ); + } + let machine = if output.stdout.iter().any(|byte| !byte.is_ascii_whitespace()) { + &output.stdout + } else { + &output.stderr + }; + serde_json::from_slice::(machine) + .with_context(|| format!("{command_name} {} did not emit JSON", row.path))?; + } + } + Ok(()) +} + +fn collect_yaml_files(directory: &Path, root: &Path, output: &mut Vec) -> Result<()> { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_yaml_files(&path, root, output)?; + } else if matches!( + path.extension().and_then(OsStr::to_str), + Some("yaml" | "yml") + ) { + output.push( + path.strip_prefix(root) + .context("example path outside repository")? + .to_string_lossy() + .replace('\\', "/"), + ); + } + } + Ok(()) +} + fn verify_docs_examples(root: &Path) -> Result<()> { let binary = binary_path(root); let examples = root.join("examples/docs"); @@ -529,6 +660,7 @@ fn verify_public_documentation(root: &Path) -> Result<()> { "docs/guides/FIRST_AGENT_WORKFLOW.md", "docs/guides/WORKFLOW_AUTHORING.md", "docs/guides/LOCAL_OPERATION.md", + "docs/guides/repair-a-failed-workflow.md", "docs/guides/CI_CD.md", "docs/guides/TROUBLESHOOTING.md", "docs/reference/YAML.md", @@ -543,6 +675,8 @@ fn verify_public_documentation(root: &Path) -> Result<()> { "docs/development/ADD_PROVIDER.md", "docs/development/ADD_MIGRATION.md", "docs/development/DOCUMENTATION.md", + "docs/execution/EXAMPLE_VERIFICATION_MATRIX.md", + "docs/research/selective-repair.md", ]; for relative in required { if !root.join(relative).is_file() { From 1e8b133f13e9325bc00dbfcdcdfd5d8dd5517889 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Thu, 23 Jul 2026 22:50:39 +0530 Subject: [PATCH 03/44] docs: document selective workflow repair --- README.md | 13 +- docs/COMPATIBILITY.md | 2 + docs/CONTAINER.md | 16 +- docs/DSL.md | 4 +- docs/DURABLE_EXECUTION.md | 7 + docs/LIMITATIONS.md | 4 + docs/OBSERVABILITY.md | 2 +- docs/OPERATIONS.md | 7 +- docs/PRODUCT.md | 5 +- docs/PROVIDERS.md | 2 + docs/SECURITY.md | 4 + docs/TESTING.md | 6 +- docs/THREAT_MODEL.md | 6 + docs/architecture/DIAGRAMS.md | 20 ++ .../SELECTIVE_REPAIR_VERIFICATION.md | 106 ++++++++++ docs/generated/CLI.md | 82 ++++++++ docs/guides/CI_CD.md | 13 +- docs/guides/LOCAL_OPERATION.md | 7 +- docs/guides/TROUBLESHOOTING.md | 18 +- docs/guides/WORKFLOW_AUTHORING.md | 4 +- docs/guides/repair-a-failed-workflow.md | 188 ++++++++++++++++++ docs/reference/CLI_OUTPUT.md | 4 +- docs/reference/DATABASE.md | 10 +- docs/reference/TERMINOLOGY.md | 8 +- docs/reference/YAML.md | 3 + docs/research/selective-repair.md | 36 ++++ examples/README.md | 4 +- 27 files changed, 555 insertions(+), 26 deletions(-) create mode 100644 docs/execution/SELECTIVE_REPAIR_VERIFICATION.md create mode 100644 docs/guides/repair-a-failed-workflow.md create mode 100644 docs/research/selective-repair.md diff --git a/README.md b/README.md index f13e371..024d585 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,16 @@ spec: message: hello from agentctl ``` -Use `check` for strict syntax, references, templates, policy, and provider-capability validation. Use `plan` for deterministic order and predictability, `run --check --diff` for a non-mutating preview, `resume` after interruption, `replay` to reconstruct recorded results without effects, and `fork` when fresh effects are intentional. +Use `check` for strict syntax, references, templates, policy, and provider-capability validation. Use `plan` for deterministic order and predictability, `run --check --diff` for a non-mutating preview, `resume` after interruption, `replay` to reconstruct recorded results without effects, `repair` to reuse compatible successful task boundaries with a corrected workflow, and `fork` when a broader fresh execution is intentional. + +Selective repair is planned before execution: + +```text +agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task --plan +agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task +``` + +See [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. ## Safety boundary @@ -62,7 +71,7 @@ CI uses the scripted fake provider. Native, mock-tested adapters cover OpenAI Re ## Repository map - `crates/agentctl-core`: DSL, compiler, templates, policy, state, effects, provider/tool contracts -- `crates/agentctl-runtime`: scheduler, actions, agent loop, resume/replay/fork +- `crates/agentctl-runtime`: scheduler, actions, agent loop, resume/replay/repair/fork - `crates/agentctl-store`: versioned SQLite persistence - `crates/agentctl-providers`: native HTTP provider adapters - `crates/agentctl-protocols`: MCP and A2A clients diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 1d853c0..96d0085 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -12,6 +12,8 @@ Unversioned `playbook:` YAML can be translated by `agentctl migrate`; `modules` `replay` now means no-effect recorded reconstruction. The prototype operation that created a new effectful run is `fork`. Unknown YAML fields, missing references, cycles, unsupported provider capabilities, invalid tool output, path escapes, unsafe processes/networks, and incompatible durable state now fail explicitly. Direct `--api-key` flags are removed; secret references are required. OpenAI uses current Responses concepts, and Anthropic/Google are native adapters rather than names on an OpenAI-compatible route. +Schema 5 adds selective-repair metadata without changing resume, replay, retry, or fork semantics. New runs persist task fingerprints, output contracts/digests, state deltas, artifacts, and disposition. Older runs migrate and remain inspectable, but tasks completed without metadata version 1 cannot be silently reused by repair. + ## Deprecated and removed Unversioned YAML is compatibility-only and warns. The TypeScript package exposes no `bin` or `main` and is archived. Placeholder memory adapters, provider environment-name-only “support,” YAML output, legacy profiles, automatic endpoint overrides, old prompt-cache fields, and optimistic replay semantics are removed from production. diff --git a/docs/CONTAINER.md b/docs/CONTAINER.md index eaa4130..14b6a70 100644 --- a/docs/CONTAINER.md +++ b/docs/CONTAINER.md @@ -26,7 +26,7 @@ The `Containerfile` combines the secret with public roots on a tmpfs mount for t Pass workflow values with repeated `--input KEY=VALUE`, `--inputs-file`, or `--inputs` JSON. Prefer files for large or sensitive non-provider inputs. Provider credentials are environment references only; never put a key in CLI arguments, YAML, an image layer, or an ordinary input value. Before a bind-mount run, provision `/state` and `/artifacts` host directories so UID/GID 65532 can write them and the runner's artifact collector can read them. Durable state may contain prompts and outputs; protect it like a sensitive build artifact. -The image emits exactly one versioned JSON result on stdout with `--output json`; failures emit one versioned JSON error on stderr. The document includes exit status semantics, run/trace IDs, final state, and declared outputs. Progress is not mixed into stdout. Persist `/state` for later `inspect`, approval resolution, `resume`, or `replay`. +The image emits exactly one versioned JSON result on stdout with `--output json`; failures emit one versioned JSON error on stderr. The document includes exit status semantics, run/trace IDs, final state, and declared outputs. Progress is not mixed into stdout. Persist `/state` for later `inspect`, approval resolution, `resume`, `replay`, or `repair`. ## Verified Docker/Podman invocation @@ -46,6 +46,20 @@ docker run --rm --read-only --user 65532:65532 \ The value form `--env OPENAI_API_KEY` forwards an already protected host variable without placing its value in the command. The credential-free container acceptance uses the same command with the fake provider and without that environment variable. +For selective repair, mount the corrected workflow under `/config`, keep the source database under `/state`, and retain any workspace artifacts required by upstream reuse. Plan without forwarding provider credentials: + +```console +docker run --rm --read-only --user 65532:65532 --network none \ + --mount type=bind,src="$PWD/config",dst=/config,readonly \ + --mount type=bind,src="$PWD/workspace",dst=/workspace,readonly \ + --mount type=bind,src="$PWD/state",dst=/state \ + ghcr.io/OWNER/agentctl:0.2.0 \ + repair /config/repaired.yaml SOURCE_RUN_ID --from failed_task --plan \ + --workspace /workspace --db /state/runtime.db --output json --color never +``` + +The execution invocation may forward only credentials required by tasks in the fresh closure. Reused tasks do not access them. The container acceptance suite executes a credential-free repair under the same non-root, read-only-root, and mounted-state contract. + ## Pipeline examples All examples use the same image/entrypoint contract. Replace the image owner/tag and arrange the four host paths using the platform's storage mechanism. Exit `3` means approval is durably pending: retain the state directory as a protected artifact or persistent volume, resolve the approval in an operator-controlled job, and resume against that same state. Discarding the state directory makes resume impossible. diff --git a/docs/DSL.md b/docs/DSL.md index 921727b..7f9d90c 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,10 +2,12 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, an optional `when`, local `vars`, typed `with` input, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. +Task output is JSON. Built-in actions own an object contract, agents can declare provider-enforced `structuredOutput`, and a task can override the complete contract with `outputSchema`. The compiler validates schemas; the runtime validates completed and selectively reused values. + Providers, action environments, and protocol headers use `{ env: NAME }` secret references. Secret names are validated and values never become the workflow document. The compiler validates missing references, duplicate tasks, cycles, task-aware templates, tool references, provider capabilities, agent limits, and sequential runtime settings before execution. Ready tasks follow declaration order. `maxConcurrency` must be `1` in this version. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index 99bfa04..fff2dab 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -6,6 +6,7 @@ SQLite is the local history and correctness boundary. Run, task, effect, approva - Resume continues the same run from durable task state. Confirmed effects are reused. A requested-but-not-started effect may execute; a started-but-unconfirmed effect fails as uncertain. - Recorded replay creates a replay record from terminal stored outputs and calls no provider, tool, network, process, or filesystem executor. +- Selective repair creates a new source-linked run, materializes compatible successful task outputs and committed state deltas, then executes selected roots and descendants with fresh effects from a target workflow. - Fork creates a new run linked to the old run and intentionally permits fresh effects. - Retry creates a new task attempt only within the task’s explicit bound. An unsafe unresolved effect is not retried. @@ -15,6 +16,12 @@ Pure operations need no external guarantee. Idempotent and keyed effects may be Working-memory replacement, the task transition, checkpoint, and audit event commit in one SQLite transaction. Tool-effect and tool-call terminal status also commit together, so inspection cannot observe one as completed while the other remains started. On resume, a confirmed memory-write effect is applied to the reconstructed working-memory value during the succeeding transition. Long-term memory is an external effect and is not rolled back by replay. +Successful task completion also commits repair metadata atomically: definition fingerprint, resolved-input digest, output-contract fingerprint, output digest, immutable state delta and digest, artifact manifest, audit event, and checkpoint. Repair initialization starts from target initial memory and applies only reused successful task deltas in topological order. It never copies a terminal source's final memory snapshot. + +Repair planning is effect-free. A source task is reusable only when its metadata version, definition, dependencies, resolved inputs, output contract/value, state delta, artifacts, and effect certainty are compatible. The repair run stores the reused result and provenance in its own task row, so later source-row garbage collection does not break it. Artifact bytes remain a separately retained workspace responsibility. + Cancellation is both an injected token and a durable run flag. CLI SIGINT and SIGTERM cancel in-flight async calls and return exit `130`; `agentctl cancel` records a request for another process to observe. An overall CLI deadline can be set with `--timeout-seconds`, in addition to task/tool/provider/protocol bounds. A provider, tool, process, MCP, or A2A timeout/cancellation/transport loss after dispatch marks the effect `uncertain`; resume refuses to guess and requires reconciliation or an explicit fork. +A repaired agent task starts a fresh provider session. Source `previous_response_id`, incomplete turns, pending tool calls, and reasoning state are not copied. Validated task output and reconstructed memory are the only cross-task/cross-run dataflow. + Clock and ID generation are injected; test providers/tools/protocol handlers are injected. The current scheduler is sequential, so output and memory commit order is task declaration order. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 7756430..c7568a1 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -44,6 +44,10 @@ These are useful extensions but are not required by the product thesis. They nee - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. - At-most-once model/remote calls can become uncertain in the dispatch/acknowledgement window. Inspect and reconcile externally; use `fork` only when fresh effects are knowingly acceptable. +- Selective repair requires task metadata version 1. Successful tasks from databases created before schema 5 remain inspectable but must execute from an earlier repair root or a full fork. +- Automatic artifact manifests cover bounded files reported by successful workspace-mutation results. Artifact bytes are not copied into SQLite or a content-addressed store; retain the configured workspace and restore by verified digest when needed. +- A confirmed non-idempotent mutation in a repair closure remains blocked. The only built-in reconciliation outcome is an operator-confirmed `not-applied` result for a started or uncertain effect; compensation and provider-specific deduplication workflows are not implemented. +- Retry remains a bounded same-run task policy. There is no separate command that creates a new terminal-source retry run for an unchanged workflow; use repair with an unchanged target definition and explicit roots when its compatibility checks fit. - Tool-using OpenAI/Azure agents require stored-response continuation. `store: false` is rejected until stateless response-item replay is implemented. - Anthropic, Google, Azure OpenAI, MCP, and A2A are native and mock-tested in this release, not live-tested. Only the OpenAI GPT-5.6 tool path has live end-to-end evidence. - The current local OCI runtime, vulnerability-scan, and SBOM evidence is Linux arm64. Linux x64 is configured in the unpushed Ubuntu workflow but has not executed. diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index 689f193..4d48adb 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -21,7 +21,7 @@ agentctl inspect RUN_ID --db .agentctl/runtime.db --output json --color never agentctl db stats --db .agentctl/runtime.db --output json --color never ``` -Inspection includes task attempts, checkpoints, effect state, approvals, provider and protocol records, ordered audit events, and trace correlation. Use `agentctl approvals list RUN_ID` when the run exited pending approval. Preserve the database and its WAL files together when the history is operational evidence. +Inspection includes task attempts, disposition, repair source/roots, per-task reuse provenance and compatibility evidence, fingerprints/digests, checkpoints, effect state, approvals, provider and protocol records, ordered audit events, and trace correlation. A reused task emits a durable `task.reused` trace event and `repair.task_reused` audit event but no fresh effect, provider-session, or tool-call row. Use `agentctl approvals list RUN_ID` when the run exited pending approval. Preserve the database and its WAL files together when the history is operational evidence. ## Runtime events diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0bf0890..6194336 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -60,7 +60,10 @@ A oneshot service has one active invocation at a time. Use distinct databases on 2. Run `agentctl inspect RUN_ID --db PATH --output json`. 3. Resolve a pending approval, then `resume`; never use `fork` as an implicit retry. 4. Use `replay` for a no-effect reconstruction of a terminal run. -5. Use `fork` for a new run that may execute fresh effects. -6. For an uncertain effect, reconcile the remote system first. The runtime intentionally refuses unsafe resume. +5. Use `repair TARGET SOURCE --from TASK --plan` before executing a corrected terminal workflow from a task boundary. +6. Use `fork` for a broader new run that may execute fresh effects. +7. For an uncertain effect, reconcile the remote system first. The runtime intentionally refuses unsafe resume or repair. + +Repair planning exits `3` when compatibility or effect safety blocks reuse. Read `blockedReuse`, choose an earlier/additional root, restore a verified artifact, or reconcile an effect. Do not bypass the plan with a fresh fork unless repeating all effects is an intentional operator decision. Use `agentctl gc --db PATH --older-than-days N` for expired memory and old terminal histories after the organization's retention/backup requirements are satisfied. SQLite WAL files belong with the database during backup. A future schedule-run key may improve deduplication; today the external scheduler owns overlap prevention. diff --git a/docs/PRODUCT.md b/docs/PRODUCT.md index ec7271f..74ebf1e 100644 --- a/docs/PRODUCT.md +++ b/docs/PRODUCT.md @@ -6,7 +6,7 @@ Primary users are application and platform engineers authoring reviewed automation, security-conscious teams introducing model calls into existing operations, CI maintainers needing credential-free validation, and Rust applications embedding the runtime. Their jobs are to validate before acting, understand an exact plan, constrain effects, recover from interruption, prove what happened, and reuse reviewed content. -Core use cases are local repository automation, approval-gated changes, structured model enrichment, provider-portable agent tasks, MCP tool calls, A2A delegation, cron-invoked runs, and generic containerized CI steps. `agentctl` is a schedulable runtime, not a scheduler: cron, systemd, Kubernetes, and CI own triggers and overlap policy. Hosted orchestration, a visual builder, chat, distributed scheduling, a public registry, arbitrary configuration management, secret storage, and unbounded autonomy are non-goals. +Core use cases are local repository automation, approval-gated changes, structured model enrichment, provider-portable agent tasks, selective repair from a failed task boundary, MCP tool calls, A2A delegation, cron-invoked runs, and generic containerized CI steps. `agentctl` is a schedulable runtime, not a scheduler: cron, systemd, Kubernetes, and CI own triggers and overlap policy. Hosted orchestration, a visual builder, chat, distributed scheduling, a public registry, arbitrary configuration management, secret storage, and unbounded autonomy are non-goals. ## Journeys @@ -14,6 +14,7 @@ Core use cases are local repository automation, approval-gated changes, structur - Scheduled: invoke the CLI without a TTY, use explicit database/workspace/artifact paths and an overall timeout, receive exit `3` for a durable pending approval, and resume through an operator-controlled invocation. - CI: mount config/workspace/state/artifacts into the generic OCI image, inject secrets only as environment variables, pass inputs by `--inputs-file` or repeated `--input`, and consume one versioned final JSON envelope on stdout. - Embedded: construct core workflow and plan values, inject a store, providers, tools, clock, IDs, and tracing, then invoke the runtime with a cancellation token. +- Repair: keep the failed terminal source immutable, compile a corrected target, plan one or more roots, reuse compatible successful boundaries, and execute only the roots and their affected descendants. Provider portability means the internal message, tool, continuation, usage, and capability contracts do not expose provider SDK types. It does not mean every provider has identical features. Compilation rejects a requested feature absent from the chosen provider. @@ -33,4 +34,4 @@ Version 0.2 is a production-oriented alpha with executable evidence for the stat This is not a chat-agent or multi-agent conversation framework: workflows, not conversations, own control flow. It is not CI/CD: it can run inside CI but does not manage runners or deployment environments. It borrows idempotence and check/diff vocabulary from Ansible without becoming configuration management. It borrows plan/effect separation from Terraform without owning infrastructure state. It is not a hosted orchestrator or general scripting language: one local process, SQLite, constrained templates, typed actions, and explicit remote effects are intentional boundaries. -The differentiator is the combination of deterministic compilation, honest predictability, durable effect identity, recorded no-effect replay, native provider portability, and policy decisions made outside the model. +The differentiator is the combination of deterministic compilation, honest predictability, durable effect identity, recorded no-effect replay, compatibility-checked task-boundary repair, native provider portability, and policy decisions made outside the model. diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index e832bdb..ebad1c9 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -24,6 +24,8 @@ Endpoints must pass the workflow network allowlist. Redirects are disabled. Cred `agentctl providers smoke-openai --live --model gpt-5.6` remains a provider-only diagnostic; it is not runtime acceptance. The repository-owned live gate is `cargo xtask acceptance-live-openai`. It runs a YAML workflow through compilation, SQLite, a real strict function call, built-in tool policy/schema validation, `previous_response_id` continuation, deterministic assertion/artifact creation, public inspection, and replay with the credential removed. It repeats the journey inside the production OCI image and never runs in normal CI. Anthropic, Google, and Azure are implemented and mock-tested but are not live-tested in this release. +`cargo xtask examples-verify-live-openai` is the broader opt-in gate. It runs every public OpenAI workflow plus the canonical two-agent repair. The repaired task starts a new Responses session and uses `previous_response_id` only between its own new turns. The failed source task's response ID, pending tool call, and reasoning state are not copied. Validated task output is the cross-run dataflow boundary. + OpenAI provider options are an allowlisted map (`store`, `reasoningContext`, `promptCacheMode`, `promptCacheTtl`, `parallelToolCalls`, and `safetyIdentifier`). Unknown options or invalid values fail compilation. Tool-using OpenAI and Azure OpenAI agents may not set `store: false`: stateless continuation would require replaying returned response/reasoning/function items, which this release does not implement. One-turn agents without tools may disable storage. Programmatic tool calling and model streaming are explicitly unsupported in the workflow runtime and fail rather than being ignored. Parallel function calls are parsed and correlated, but executors run them serially in response order because v1 scheduling is sequential. Cost is not inferred when a provider returns no reliable cost metadata. A workflow requesting `maxCostUsd` therefore fails capability negotiation; input/output token limits are enforced from native usage. Retry is limited to explicit task bounds and definitive retryable HTTP responses. Timeout, cancellation, or a transport loss after dispatch is considered ambiguous and is not automatically reissued. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 606ddf3..d870168 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -10,6 +10,8 @@ - Tool input and output JSON Schemas are enforced. Models, MCP annotations, A2A cards, remote schemas, and results cannot grant capabilities. - Requests are ledgered before effects. Global denial or approval cannot be weakened by a tool contract. Approval is durable; non-interactive mode pauses with exit `3` or uses an explicitly stricter deny/fail mode, never a prompt or implicit approval. - SQLite uses foreign keys, WAL/busy timeout, version checks, checksummed checkpoints, and mode `0600` on Unix. +- Repair never mutates a terminal source. Reuse requires versioned definition/input/contract/output/state metadata and verified artifact paths, sizes, and SHA-256 digests. Repair creation and reused-task materialization are one SQLite transaction. +- A recorded replay cannot be a repair source because it has no direct effect ledger. A materialized reused/recorded task cannot be selected for restart without returning to direct effect history. Repaired agents start fresh provider sessions. - Packs require a supported manifest/version and can be checked against SHA-256 integrity. - The workspace forbids unsafe Rust, denies warnings, locks dependencies, checks licenses/sources/advisories, scans secret patterns, and keeps live tests outside CI. @@ -21,4 +23,6 @@ Prompts, file content, model output, remote artifacts, and tool output may be co MCP reconnection and A2A resubmission are intentionally not automatic. Streaming is bounded but completed results, not token deltas, enter workflow state. Windows cannot express Unix database mode bits; rely on the user profile ACL and CI tests. +SQLite file access is the repair authorization boundary. There is no tenant identity or row-level authorization. Artifact bytes remain in the workspace rather than SQLite; losing or changing them blocks reuse but does not restore them automatically. + Report vulnerabilities privately to the repository maintainer. Do not include credentials, database contents, or production prompts in a report. diff --git a/docs/TESTING.md b/docs/TESTING.md index 766fc71..2dfbf42 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -12,13 +12,15 @@ User-journey layers are separate: cargo xtask acceptance cargo xtask acceptance-container cargo xtask acceptance-live-openai # explicit credentialed gate only +cargo xtask examples-verify +cargo xtask examples-verify-live-openai # explicit credentialed gate only cargo xtask package cargo xtask secret-scan ``` It checks rustfmt; clippy with all targets/features and warnings denied; locked build; unit, integration, compatibility, provider, protocol, persistence, runtime, and security tests; rustdoc; generated schema/CLI consistency; all workflow validation and deterministic examples; negative capability/policy/no-mutation cases; dependency sources/licenses/advisories; repository secret patterns and immutable workflow action pins; `cargo install`; and the Rust-only production boundary. -Unit tests cover parser diagnostics, strictness, compiler order/cycles/capabilities, templates, tool schemas, policy traversal/network/redaction, state transitions, effect recovery, store migration/corruption/checkpoints, runtime dataflow/check/diff/approval/cancellation/replay/fork, provider mappings, protocols, and traces. `proptest` exercises arbitrary templates and typed preservation. Language-neutral fixtures in `fixtures/compat` preserve the TypeScript oracle’s external graph/dataflow contract. +Unit tests cover parser diagnostics, strictness, compiler order/cycles/capabilities, templates, tool schemas, policy traversal/network/redaction, state transitions, effect recovery, store migration/corruption/checkpoints, runtime dataflow/check/diff/approval/cancellation/replay/repair/fork, provider mappings, protocols, and traces. Repair regressions cover two-agent reuse with a panic-on-repeat provider, downstream and branch closure, repeated roots, changed definitions/prompts, output/state/artifact corruption, migration and rollback, effect uncertainty/reconciliation, approval gating, source garbage collection, and effect-free replay. `proptest` exercises arbitrary templates and typed preservation. Language-neutral fixtures in `fixtures/compat` preserve the TypeScript oracle’s external graph/dataflow contract. `fuzz/` contains `cargo-fuzz` targets for workflow YAML/templates, provider responses, MCP/A2A payload shapes, persisted state, and tool schemas/inputs. They use no network or credentials. Example: @@ -29,4 +31,4 @@ cargo fuzz run workflow_yaml -- -max_total_time=60 The local hosted-CI configuration runs the canonical suite, credential-free acceptance, and packaging on Rust 1.88 for Linux x64, macOS arm64, and Windows x64. Separate automatic jobs cover the Linux x64 container, current vulnerability scan, two CycloneDX SBOM artifacts, complete-history/tree secret scans, dependency policy, and workflow lint. The workflows are locally linted but have not been pushed or dispatched, so this is configured evidence rather than validated hosted-platform support. Provider/protocol conformance uses local mock HTTP servers. Normal examples are deterministic; MCP/A2A runtime behavior is covered by mocks rather than requiring a background service. -The only full live gate is the separately invoked OpenAI acceptance described in [Providers](PROVIDERS.md). It performs two bounded Responses API requests locally and two in the OCI image for one tool-call/continuation journey each, then performs keyless replays. Never run it for debugging loops, fuzzing, load, or normal CI. +Live gates are separately invoked and described in [Providers](PROVIDERS.md). The original acceptance performs one tool-call/continuation journey locally and in the image. `examples-verify-live-openai` inventories and runs every OpenAI-backed example, including the failed two-agent source, selective repair, and keyless replay, with a 40-request and conservative USD 10 guard. Never run either command for debugging loops, fuzzing, load, or normal CI. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 8771c16..87deee6 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -17,9 +17,15 @@ The local operator and reviewed binary are trusted. Workflow authors are only as | MCP annotation or A2A card claims safety | always treated as untrusted metadata | compromised authorized peer can return malicious but schema-valid data | | Crash duplicates an external mutation | request-before-start ledger, uncertain state, no silent retry | external action may have happened without acknowledgement | | Replay reissues effects | recorded replay uses stored terminal output only | replayed data may no longer reflect current reality, by design | +| Repair reuses tampered or unrelated state | stable workflow identity, versioned task/input/contract/output/state fingerprints, artifact digest checks, transactional materialization | an attacker with database/workspace write access is inside the local application trust boundary | +| Repair duplicates a partial mutation | closure effect inspection, conservative uncertainty block, narrow operator `not-applied` reconciliation | remote truth may remain unknowable and keep the repair blocked | +| Repair carries failed model state | every repaired agent starts a fresh provider session; dataflow uses validated JSON output | a valid reused output can still contain hostile content and must remain policy constrained | +| Source deletion breaks repair | reused output/state/artifact metadata is materialized into the repair run | artifact bytes still require durable workspace retention | | Approval bypass in CI | non-interactive durable pause or explicit deny/fail; operator resolution | stolen database write access is outside application trust boundary | | Pack substitution | SHA-256 verification and semver/API checks | digest source/signature trust is manual | | Corrupt or future state misexecutes | schema/version/checksum/deserialization failures | SQLite file deletion or rollback by an attacker is not prevented | | Dependency compromise | locked registry-only deps, cargo-deny, license/source checks | registry compromise and zero-days remain possible | No unresolved critical or high-severity defect is knowingly accepted for the implemented boundary. Deferred sandboxing, signature verification, distributed concurrency, and encrypted storage are explicit product limitations, not implied controls. + +Run access control is the database file and operating-system identity. `agentctl` has no multi-tenant authorization layer; do not let an untrusted principal select another tenant's source run from a shared database. diff --git a/docs/architecture/DIAGRAMS.md b/docs/architecture/DIAGRAMS.md index 8f2e94e..6d408f7 100644 --- a/docs/architecture/DIAGRAMS.md +++ b/docs/architecture/DIAGRAMS.md @@ -143,6 +143,26 @@ flowchart LR Replay reports historical truth. It does not observe current files, rerun verification, or contact a provider. +## Selective repair flow + +Selective repair is a new source-linked run. It is distinct from both effect-free recorded replay and broad fresh fork execution. + +```mermaid +flowchart LR + accTitle: Selective repair flow + accDescr: Repair verifies successful upstream task boundaries, materializes compatible outputs and state, and executes selected roots and descendants from a target workflow. + Source[Terminal source run] --> Plan[Effect-free compatibility plan] + Target[Target workflow] --> Plan + Plan --> Reuse[Materialize compatible upstream tasks] + Plan --> Fresh[Execute roots and descendants] + Reuse --> Boundary[Reconstructed task-boundary state] + Boundary --> Fresh + Fresh --> Repair[New repair run and trace] + Source -. remains immutable .-> Repair +``` + +The detailed failed-run, plan, reuse, invalidation, lineage, and effect-safety diagrams are in [Repair a failed workflow](../guides/repair-a-failed-workflow.md). + ## Fork or rerun flow Fork makes fresh execution an explicit choice instead of overloading replay. diff --git a/docs/execution/SELECTIVE_REPAIR_VERIFICATION.md b/docs/execution/SELECTIVE_REPAIR_VERIFICATION.md new file mode 100644 index 0000000..6f3b6a3 --- /dev/null +++ b/docs/execution/SELECTIVE_REPAIR_VERIFICATION.md @@ -0,0 +1,106 @@ +# Selective repair verification + +This record contains sanitized release evidence for selective workflow repair. It +does not contain provider responses, credentials, runtime databases, or prompt +transcripts. + +## Baseline + +Before implementation: + +- `cargo xtask verify` passed. +- `cargo xtask acceptance` passed 25 credential-free scenarios. +- `cargo xtask package` passed. +- `cargo xtask acceptance-container` was blocked because the installed Podman + engine could not connect to its Linux VM. + +## Deterministic release gates + +On 2026-07-23: + +- `cargo xtask verify` passed all 12 stages, including formatting, Clippy with + warnings denied, workspace tests, documentation tests, dependency policy, + secret and action-pin scans, source installation, and the production boundary. +- `cargo xtask acceptance` passed all 28 scenarios. +- `cargo xtask examples-verify` passed the complete discovered example matrix. +- `cargo xtask docs-verify` passed all six stages. +- `cargo xtask package` produced the macOS arm64 package. + +The deterministic repair coverage includes task-boundary reuse, downstream and +branch invalidation, multiple roots, changed definitions and prompt files, +output-contract and digest failures, missing artifacts, state reconstruction, +effect reconciliation, idempotency, approval gates, source immutability, source +garbage collection, migrations, transaction rollback, stable JSON output, and +effect-free offline replay. + +## Live OpenAI evidence + +Command: + +```console +cargo xtask examples-verify-live-openai +``` + +Local packaged-CLI verification completed for all six OpenAI example workflows +with model `gpt-5.6`: + +- Requests: 10 +- Tool calls: 4 +- Input tokens: 1,894 +- Output tokens: 214 +- Reasoning tokens: 0 +- Prompt-cache read tokens: 0 +- Prompt-cache write tokens: 0 +- Estimated standard-tier model cost at the verified 2026-07-23 public rates: + USD 0.01589 + +Selective-repair lineage: + +- Source run: `run-019f8ffa-907c-7a41-944b-d6c303f898ec` +- Repair run: `repair-019f8ffa-a3c9-72a0-9a8a-2bf82e1566fb` +- Offline replay: `replay-019f8ffa-b426-78b0-b2c8-1d9b113b2824` + +The live assertions proved: + +- The source `analyze` task succeeded and `publish` failed after live agent and + tool execution. +- The plan marked `analyze` reused and `publish` executable. +- The repair run contained no fresh effect, provider session, or tool call for + `analyze`. +- `publish` used one fresh task-local provider session and one model-selected + tool call; it did not continue the failed source session. +- The repaired output and artifact contained the persisted upstream marker + `SELECTIVE_REPAIR_FIXTURE_CONFIRMED`, proving the repaired task consumed the + reused result. +- The source run and source task records were unchanged after repair. +- Replay ran after removing `OPENAI_API_KEY`, preserved semantic output and the + artifact digest, and dispatched zero effects, tool calls, or provider sessions. + +The ignored local evidence file is +`.release-evidence/selective-repair/live-summary.json`. + +## Blocked container gates + +These commands remain blocked by the local container runtime: + +```console +env -u OPENAI_API_KEY cargo xtask acceptance-container +cargo xtask examples-verify-live-openai +``` + +Podman 5.8.2 is installed, but its `libkrun` VM socket at +`127.0.0.1:52210` refuses connections. The live command therefore completed +the local packaged-CLI stage and stopped before building or running the OCI +stage. No container result is claimed. + +Continuation: + +```console +podman machine start +env -u OPENAI_API_KEY cargo xtask acceptance-container +cargo xtask examples-verify-live-openai +``` + +The final command would repeat the already completed local live calls before +reaching its container phase, so rerun it only when the remaining request +budget and cost are explicitly accepted. diff --git a/docs/generated/CLI.md b/docs/generated/CLI.md index a0b7c41..307063f 100644 --- a/docs/generated/CLI.md +++ b/docs/generated/CLI.md @@ -16,8 +16,10 @@ Commands: resume Continue an interrupted or approval-paused run replay Reconstruct a terminal run only from recorded state and results fork Create a new run from a prior workflow with fresh effects + repair Create a new run that reuses compatible upstream results and executes a repaired suffix cancel Durably request cancellation inspect Inspect durable run, task, and audit state + effects Inspect or narrowly reconcile uncertain effects approvals List or resolve durable approval requests providers Inspect provider capabilities or run the opt-in OpenAI smoke auth Check configured secret references without revealing values @@ -161,6 +163,33 @@ Options: -h, --help Print help ``` +## `agentctl repair` + +```text +Create a new run that reuses compatible upstream results and executes a repaired suffix + +Usage: agentctl repair [OPTIONS] --from + +Arguments: + + + +Options: + --from + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --plan + --restart-successful + --verbose + --reason + --db [default: .agentctl/runtime.db] + --interactive + --diff + --workspace + --timeout-seconds + -h, --help Print help +``` + ## `agentctl cancel` ```text @@ -197,6 +226,59 @@ Options: -h, --help Print help ``` +## `agentctl effects` + +```text +Inspect or narrowly reconcile uncertain effects + +Usage: agentctl effects [OPTIONS] + +Commands: + inspect + reconcile + +Options: + --db [default: .agentctl/runtime.db] + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl effects inspect` + +```text +Usage: agentctl effects inspect [OPTIONS] + +Arguments: + + +Options: + --output [default: human] [possible values: human, json] + --task + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl effects reconcile` + +```text +Usage: agentctl effects reconcile [OPTIONS] --outcome --reason + +Arguments: + + +Options: + --outcome [possible values: not-applied] + --output [default: human] [possible values: human, json] + --actor [default: cli-user] + --color [default: auto] [possible values: auto, always, never] + --reason + --verbose + -h, --help Print help +``` + ## `agentctl approvals` ```text diff --git a/docs/guides/CI_CD.md b/docs/guides/CI_CD.md index 25bdb53..d85f964 100644 --- a/docs/guides/CI_CD.md +++ b/docs/guides/CI_CD.md @@ -10,7 +10,7 @@ Every platform uses the same paths: | --- | --- | --- | | `/config` | read-only | reviewed workflow, inputs, and packs | | `/workspace` | normally read-only | checked-out source and fixtures | -| `/state` | writable and retained | SQLite database, resume, replay, approvals | +| `/state` | writable and retained | SQLite database, resume, replay, repair, approvals | | `/artifacts` | writable and collected | declared reports and outputs | Use `--output json --color never`. A successful workflow exits `0`. Validation exits `2`; policy or a pending approval exits `3`; run failure exits `4`; persistence exits `5`; provider or protocol failure exits `6`; cancellation exits `130`. @@ -48,6 +48,17 @@ Mount an ordinary JSON file under `/config` and pass `--inputs-file /config/inpu A non-interactive approval does not wait for stdin. It persists a request, exits `3`, and requires the same `/state` data in a later operator-controlled job. That job lists and resolves the approval, then calls `resume`. If your pipeline cannot retain protected state between jobs, configure policy to deny or fail instead of using approvals. +## Selective repair in pipelines + +Keep the failed terminal `/state` and durable workspace, publish a reviewed corrected workflow, and run an effect-free planning step first: + +```text +agentctl repair /config/repaired.yaml SOURCE_RUN_ID --from failed_task --plan \ + --workspace /workspace --db /state/runtime.db --output json --color never +``` + +Permit the execution step only when the plan exits `0` and the machine output's source run, target digest, roots, fresh effects, and approvals match the review. Exit `3` can also mean a blocked repair plan, so distinguish `kind: RepairPlan` from a pending run approval. Retain the new repair run ID as independent audit evidence. + ## Retention and recovery Collect `/state` even on failure when recovery or audit matters. It can contain confidential prompts and outputs, so apply protected artifact access and a short, documented retention period. Keep `/artifacts` according to the report's classification. diff --git a/docs/guides/LOCAL_OPERATION.md b/docs/guides/LOCAL_OPERATION.md index fca2fac..8dfa650 100644 --- a/docs/guides/LOCAL_OPERATION.md +++ b/docs/guides/LOCAL_OPERATION.md @@ -1,6 +1,6 @@ # Operate agentctl locally -Use explicit paths and retain the database whenever you may need inspection, approval, resume, replay, or audit evidence. +Use explicit paths and retain the database whenever you may need inspection, approval, resume, replay, repair, or audit evidence. ## Default and custom paths @@ -34,14 +34,15 @@ agentctl approvals list RUN_ID --db /var/lib/agentctl/runtime.db --output json - Use the run ID and trace ID when correlating logs. Treat database output as sensitive because prompts, file content, tool output, and remote artifacts may be present even when secret values were redacted. -## Resume, replay, retry, and fork +## Resume, replay, retry, repair, and fork - Resume continues the same non-terminal run and reuses confirmed effects. - Retry is bounded within a task and never guesses about an ambiguous effect. - Recorded replay creates a new record from terminal stored results and calls no executor. +- Repair creates a new source-linked run, reuses only compatible successful tasks before selected roots, and executes every root and descendant from a supplied target workflow. - Fork creates a new child run and permits fresh effects. -Do not use these terms interchangeably. Read [Durable execution](../DURABLE_EXECUTION.md) before recovering a workflow that may have changed an external system. +Do not use these terms interchangeably. Read [Durable execution](../DURABLE_EXECUTION.md) before recovering a workflow that may have changed an external system. For a corrected terminal workflow, follow [Repair a failed workflow](repair-a-failed-workflow.md). ## Resolve an approval diff --git a/docs/guides/TROUBLESHOOTING.md b/docs/guides/TROUBLESHOOTING.md index 34152eb..746368f 100644 --- a/docs/guides/TROUBLESHOOTING.md +++ b/docs/guides/TROUBLESHOOTING.md @@ -12,7 +12,7 @@ flowchart TD D --> E{Pending approval?} E -->|Yes| F[Review and resolve approval, then resume] E -->|No| G{Uncertain effect?} - G -->|Yes| H[Reconcile the external system before any fork] + G -->|Yes| H[Reconcile the external system before repair or fork] G -->|No| I[Use task, effect, provider, and audit evidence] ``` @@ -109,6 +109,22 @@ ls -ld /state /state/runtime.db **Resolve:** Resume only a safe non-terminal run. Replay only a terminal run. Reconcile uncertain external state before an explicit fork. +## Repair plan blocked + +**Symptom:** `repair --plan` emits a valid `RepairPlan` with `compatible: false` and exits `3`. + +**Diagnose:** + +```text +agentctl repair target.yaml SOURCE_RUN_ID --from TASK --plan \ + --db .agentctl/runtime.db --output json --color never +agentctl effects --db .agentctl/runtime.db inspect SOURCE_RUN_ID --task TASK +``` + +**Expected evidence:** Each `blockedReuse` item names the task, compatibility rule, safe source/target fingerprints, suggested root, and whether a full fork is required. + +**Resolve:** Choose the earliest changed/incompatible producer as another repair root, restore the exact verified artifact, add a structured output contract and create a fresh source result, or reconcile an uncertain effect only after checking external reality. Do not edit task rows or use fork as a generic force option. See [Repair a failed workflow](repair-a-failed-workflow.md). + ## Container permission or read-only failure **Symptom:** The image cannot create `/state/runtime.db` or write `/artifacts`. diff --git a/docs/guides/WORKFLOW_AUTHORING.md b/docs/guides/WORKFLOW_AUTHORING.md index 9258fd3..3b46904 100644 --- a/docs/guides/WORKFLOW_AUTHORING.md +++ b/docs/guides/WORKFLOW_AUTHORING.md @@ -85,6 +85,8 @@ agents: The model does not own the graph, policy, or persistence. +When an agent result feeds another task, declare `structuredOutput` as a JSON Schema. It becomes the task's durable output contract and lets selective repair verify and reuse the result. A task-level `outputSchema` is available when the complete task contract must differ from the agent or action default. + ## 7. Define tool contracts A model sees only tools listed on its agent. Each tool requires strict input and output schema, capability, risk, effect class, idempotency, retry safety, timeout, and approval requirement. Runtime policy makes the final authorization decision. @@ -108,7 +110,7 @@ Start with the minimum grant. Add a host, writable root, executable, or secret n ## 10. Plan for state and recovery -Choose an explicit database path for scheduled or CI runs. A confirmed effect can be reused during resume. An effect that started without a confirmed result becomes uncertain and stops automatic recovery. Recorded replay calls no executor. Fork intentionally permits fresh effects. +Choose an explicit database path for scheduled or CI runs. A confirmed effect can be reused during resume. An effect that started without a confirmed result becomes uncertain and stops automatic recovery. Recorded replay calls no executor. Repair can reuse compatible successful task boundaries and execute a corrected suffix. Fork intentionally permits a broader fresh execution. ## Validate your workflow diff --git a/docs/guides/repair-a-failed-workflow.md b/docs/guides/repair-a-failed-workflow.md new file mode 100644 index 0000000..5f74f10 --- /dev/null +++ b/docs/guides/repair-a-failed-workflow.md @@ -0,0 +1,188 @@ +# Repair a failed workflow + +Suppose `analyze` and `publish` are agent tasks. `analyze` succeeded and stored validated JSON. `publish` called its read-only tool but failed because its turn limit was too small. You corrected only `publish`. + +Do not resume the failed terminal run. Resume continues the same non-terminal run with the same compiled definition. Do not replay it to execute the fix. Recorded replay copies terminal recorded results and emits no fresh effects. Use repair to create a linked run that reuses compatible `analyze` data and executes `publish` plus its descendants from the corrected workflow. + +The runnable example is in [`examples/selective-repair-openai/`](../../examples/selective-repair-openai/README.md). + +## 1. Understand the failed source + +```mermaid +flowchart LR + A["analyze
succeeded
structured output stored"] --> B["publish
failed after tool call"] + B -. blocked .-> C["verify
not run"] + C -. blocked .-> D["artifact
not written"] +``` + +Inspect the failed run: + +```bash +agentctl inspect SOURCE_RUN_ID \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` + +Confirm that the source is terminal, `analyze` is `succeeded`, `publish` is `failed`, and any partial effects are understood. + +## 2. Fix task 2 and plan + +Increase the turn limit or correct the task instructions, prompt file, tool configuration, implementation, timeout, or output contract in the target workflow. Then plan without calling a provider or tool: + +```bash +agentctl repair repaired.workflow.yaml SOURCE_RUN_ID \ + --from publish \ + --plan \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` + +```mermaid +flowchart TD + S["Terminal source run"] --> C["Compile source and target graphs"] + C --> R["Roots: publish"] + R --> D["Closure: publish, verify, artifact"] + C --> U["Candidate reuse: analyze"] + U --> K{"All compatibility checks pass?"} + K -->|yes| P["Plan: analyze reused"] + K -->|no| X["Block before run creation"] + D --> E["Plan: closure executes freshly"] +``` + +The plan reports source and target workflow digests, roots, reused and rerun tasks, new/removed/changed tasks, blocked checks, estimated provider tasks, fresh effects, and possible approvals. A compatible plan exits `0`. A blocked plan is still valid JSON and exits `3`. + +For independent failed branches, repeat the root: + +```bash +agentctl repair workflow.yaml SOURCE_RUN_ID \ + --from analyze_a \ + --from analyze_b \ + --plan +``` + +A source task that already succeeded can be a fresh root only with `--restart-successful`. + +## 3. How upstream reuse works + +```mermaid +flowchart LR + SO["Source analyze result"] --> V["Verify metadata v1"] + V --> F["Definition and prompt fingerprint"] + F --> I["Resolved input and dependency digest"] + I --> O["Output contract and output digest"] + O --> M["State delta digest"] + M --> A["Artifact path, size, SHA-256"] + A --> N["Materialize succeeded/reused task
attempt 0, source provenance"] +``` + +The runtime starts from target initial memory, visits reusable tasks in deterministic topological order, materializes their outputs, and applies only their committed successful state deltas. It does not copy the source run's final memory snapshot. Failed task-local state and invalidated downstream state are excluded. + +Agent tasks that feed downstream tasks need an explicit structured output contract through agent `structuredOutput` or task `outputSchema`. Built-in actions use their runtime-owned JSON output contract when a more specific schema is not needed. Outputs are validated at completion and again before reuse. + +## 4. Downstream invalidation + +```mermaid +flowchart TD + P["prepare"] --> A["analyze_a"] + P --> B["analyze_b
repair root"] + A --> C["combine"] + B --> C + classDef reused fill:#e8f5e9,stroke:#2e7d32 + classDef fresh fill:#fff3e0,stroke:#ef6c00 + class P,A reused + class B,C fresh +``` + +Every root and transitive descendant executes. Tasks outside that union are candidates for reuse, not automatically reusable. A new descendant executes. A new unrelated task blocks and asks for another or earlier root. A removed unreferenced task is reported but does not block. + +## 5. Execute and inspect + +```bash +agentctl repair repaired.workflow.yaml SOURCE_RUN_ID \ + --from publish \ + --reason "raise publish turn limit after read-only call" \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` + +```mermaid +flowchart LR + S["Source run
failed, immutable"] -->|sourceRunId| R["Repair run
new run and trace IDs"] + S1["source analyze attempt 1"] -->|provenance| R1["repair analyze
succeeded / reused / attempt 0"] + R1 --> R2["repair publish
succeeded / executed"] + R2 --> R3["repair descendants
executed"] +``` + +The result includes the new repair run ID, source run ID, trace ID, reused tasks, executed tasks, final state, and workflow outputs. `inspect` exposes run lineage and each task's disposition, source attempt, fingerprints, output/state/artifact digests, and reuse decision. Reused tasks create no provider session, tool call, process, network call, or effect row in the repair run. + +The repair run materializes reused task output and state metadata in its own rows. Deleting the source database rows later does not break repair inspection or recorded replay. Artifact bytes must remain in the configured durable workspace and are verified before reuse. + +## 6. Understand fresh-effect safety + +```mermaid +flowchart TD + E["Prior effect in repair closure"] --> C{"Effect class"} + C -->|model, observe, pure| F["Fresh execution permitted"] + C -->|mutation or remote action| S{"Recorded outcome"} + S -->|confirmed idempotent| F + S -->|failed before dispatch| F + S -->|started or uncertain| B["Block repair"] + S -->|confirmed non-idempotent| B + B --> I["Inspect effect and reconcile external reality"] + I --> N{"Confirmed not applied?"} + N -->|yes| R["Reconcile as not-applied, then re-plan"] + N -->|no or unknown| H["Choose a safe business remediation
or broader fresh execution"] +``` + +Inspect effects for the failed boundary: + +```bash +agentctl effects --db .agentctl/runtime.db inspect SOURCE_RUN_ID --task publish +``` + +If an effect is `started` or `uncertain` and an operator has verified that it did not happen: + +```bash +agentctl effects --db .agentctl/runtime.db reconcile EFFECT_ID \ + --outcome not-applied \ + --reason "remote system confirms no record" \ + --actor operator-name +``` + +There is no generic force option and no exactly-once claim. Confirmed non-idempotent effects stay blocked because repeating them may duplicate external work. Normal policy, approval, timeout, retry, and cancellation behavior applies to every fresh task. + +A repaired agent begins a new provider session. It receives target instructions and tools plus validated upstream output and reconstructed memory. It never receives the failed source task's `previous_response_id`, incomplete turn, pending call, or reasoning state. Within the new repaired task, normal multi-turn continuation still applies. + +## 7. Replay the repaired result offline + +After a successful repair: + +```bash +env -u OPENAI_API_KEY agentctl replay REPAIR_RUN_ID \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` + +Recorded replay has a new replay run ID but the same semantic outputs. It dispatches zero fresh effects and does not rewrite artifacts. + +## Troubleshooting blocked plans + +| Block | Meaning | Next action | +| --- | --- | --- | +| `repair_root_missing` | The root is absent from the target graph. | Correct the task ID or workflow. | +| `successful_root_requires_acknowledgement` | The selected root succeeded. | Add `--restart-successful` only when fresh execution is intended. | +| `definition_fingerprint_mismatch` | A task changed outside the rerun closure. | Choose that task as an earlier/additional root. | +| `resolved_input_digest_mismatch` | Inputs, dependency output, or boundary memory changed. | Choose the first affected task as a root. | +| `missing_output_contract` | A reused agent feeds downstream work without typed output. | Add structured output and create a fresh source result. | +| `output_contract_mismatch` | The target expects a different contract. | Rerun from the producer. | +| `output_digest_mismatch` | Persisted output was modified or corrupted. | Do not reuse it; rerun from the producer. | +| `artifact_integrity` | An artifact is missing, changed, or outside policy. | Restore the verified artifact or rerun its producer. | +| `legacy_task_metadata` | The source predates repair metadata v1. | Use an earlier root or a full fork. | +| `new_task_outside_repair_closure` | A new unrelated task has no result. | Add it as a root or choose an earlier common boundary. | +| `unreconciled_effect` | Fresh execution may duplicate a mutation. | Inspect and reconcile external reality first. | + +`retry` remains a task's bounded same-definition attempt policy in v1alpha1; there is no separate terminal-run `retry` command yet. Use repair for a changed target definition and fork for a broader intentionally fresh execution. diff --git a/docs/reference/CLI_OUTPUT.md b/docs/reference/CLI_OUTPUT.md index 051bdc4..384c2f2 100644 --- a/docs/reference/CLI_OUTPUT.md +++ b/docs/reference/CLI_OUTPUT.md @@ -20,7 +20,7 @@ JSONL progress output is not implemented in this release. Event-level informatio | --- | --- | --- | | `0` | success | Collect outputs and artifacts. | | `2` | usage or validation | Correct arguments, YAML, references, templates, or capabilities. | -| `3` | policy or approval | Inspect denial or retain state for operator approval. | +| `3` | policy, approval, or blocked repair plan | Inspect denial/compatibility evidence or retain state for operator approval. | | `4` | run failure | Inspect the failed task and effect history. | | `5` | persistence | Check database compatibility, permissions, corruption, and locking. | | `6` | provider or protocol | Diagnose authentication, network, native API, MCP, or A2A evidence. | @@ -28,6 +28,8 @@ JSONL progress output is not implemented in this release. Event-level informatio Do not automatically retry every nonzero code. A provider, protocol, process, or tool operation may be uncertain after dispatch. +`repair --plan` returns kind `RepairPlan`. A compatible plan exits `0`; a blocked plan exits `3` while remaining a successful, parseable machine envelope with `compatible: false` and `blockedReuse` explanations. Repair execution returns kind `RepairOutcome` with new/source run IDs, trace ID, state, reused tasks, executed tasks, and output. `inspect` exposes the complete run and task lineage. + ## Example ```text diff --git a/docs/reference/DATABASE.md b/docs/reference/DATABASE.md index 083b1a3..ce2d142 100644 --- a/docs/reference/DATABASE.md +++ b/docs/reference/DATABASE.md @@ -1,11 +1,11 @@ # Runtime database and migrations -The local SQLite database is both history and part of the correctness boundary. The current database schema version is `4`. +The local SQLite database is both history and part of the correctness boundary. The current database schema version is `5`. ## Stored records -- runs, source workflow, compiled plan, inputs, output, mode, state, and parent linkage -- task states, attempts, output, and errors +- runs, source workflow, compiled plan, inputs, output, mode, state, parent linkage, and repair source/root metadata +- task states, attempts, output, errors, disposition, source attempt, versioned fingerprints/digests, state delta, artifact manifest, and reuse decision - effects, request/result/error, confirmation, and uncertainty - approvals and resolutions - checksummed checkpoints @@ -15,6 +15,10 @@ The local SQLite database is both history and part of the correctness boundary. Working memory is stored on the run and in checkpoints. Provider credentials are not stored. Other confidential content may be stored, including prompts, tool output, and remote artifacts. +Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. A repair transaction creates the run, materializes every reused task, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete a repair run. + +Artifact manifests contain policy-resolved paths, byte sizes, and SHA-256 digests. The bytes remain in the configured durable workspace. Retain that workspace for as long as a task result may be repaired or audited. + ## Migrations The store reads SQLite `user_version` and applies forward migrations in order inside transactions. A database newer than the binary fails explicitly. Corrupt or incompatible serialized state also fails explicitly. diff --git a/docs/reference/TERMINOLOGY.md b/docs/reference/TERMINOLOGY.md index 72d7189..4f7412e 100644 --- a/docs/reference/TERMINOLOGY.md +++ b/docs/reference/TERMINOLOGY.md @@ -11,17 +11,19 @@ Use these terms consistently in workflows, documentation, issues, and reviews. | Tool | A strict capability contract that an agent may request. | | Provider | A native model API adapter behind provider-neutral contracts. | | Effect | A durably identified operation that observes or changes state outside pure computation. | -| Run | One durable execution, check, replay, or fork record. | +| Run | One durable execution, check, replay, repair, or fork record. | | Attempt | One bounded execution attempt for a task. | | Resume | Continue the same non-terminal run using durable progress. | | Recorded replay | Create a new record from terminal stored results without calling executors. | | Retry | Start another bounded attempt for a task after a definitive retry-safe failure. | +| Repair | Create a linked run from a terminal source, reuse compatible successful tasks outside selected boundaries, and execute the roots and descendants from a target workflow. | | Fork | Create a child run that intentionally permits fresh effects. | -| Rerun | Informal term. Prefer fork when referring to the supported fresh-run operation. | +| Rerun | Informal term. Prefer repair for boundary selection or fork for a broader fresh execution. | +| Disposition | Whether a successful task was freshly `executed`, source-linked `reused`, or copied as `recorded` replay evidence. | | Approval | A durable operator decision required before an effect may continue. | | Checkpoint | A versioned, checksummed snapshot used for recovery. | | Working memory | One run-local JSON object changed by explicit memory actions. | | Long-term memory | Namespaced SQLite values shared across runs and managed by retention. | | Pack | A local versioned manifest and reviewed reusable content with integrity checking. | -Do not use resume, replay, retry, and fork interchangeably. None of them means exactly-once execution. +Do not use resume, replay, retry, repair, and fork interchangeably. None of them means exactly-once execution. diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index 3e3b0c4..c9343bf 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -44,6 +44,7 @@ Each task requires `id` and `uses`. `uses` is `action:name` or `agent:name`. | `when` | true | Constrained boolean/equality expression. | | `vars` | `{}` | Task-local JSON values. | | `with` | `{}` | Typed action or agent input. | +| `outputSchema` | action-owned object or agent structured contract | Valid JSON Schema checked at task completion and selective-repair reuse. | | `retry` | bounded default | Only definitive retry-safe failures may repeat. | | `timeoutSeconds` | action or agent default | Must be within the implementation bound. | | failure behavior | fail | Unsupported dynamic control flow is rejected. | @@ -54,6 +55,8 @@ Ready tasks run in YAML declaration order. There is no `foreach`, matrix, loop, An agent requires `provider` and `model`. Defaults are `maxTurns: 8`, `maxToolCalls: 16`, `maxOutputTokens: 2048`, and `timeoutSeconds: 120`. Set tighter values for known work. Optional fields include instructions or `instructionsFile`, variables, tools, retry, reasoning, structured output, usage limits, and provider-specific options. +`structuredOutput` asks the provider for typed JSON and becomes the default task output contract. A task-level `outputSchema` can define the complete task contract explicitly. An agent result that feeds downstream tasks must have one of these contracts before it can be reused by selective repair. Schema documents are compiled when the workflow is checked; values are validated both when completed and when reused. + Capability negotiation happens during compilation. A provider must explicitly support every requested feature. ## Actions diff --git a/docs/research/selective-repair.md b/docs/research/selective-repair.md new file mode 100644 index 0000000..c9d22c4 --- /dev/null +++ b/docs/research/selective-repair.md @@ -0,0 +1,36 @@ +# Selective repair research + +Status: implemented for `agentctl` v1alpha1. Reviewed against primary material on 2026-07-23. + +Selective repair is a new execution from a task boundary, not history replay. The design borrows narrow safety patterns from workflow engines and build systems without adopting their execution models. + +| Problem solved | Pattern considered | Decision | Reason | `agentctl` consequence | Safety consequence | Compatibility consequence | +| --- | --- | --- | --- | --- | --- | --- | +| Keep recorded replay deterministic | [Temporal workflow replay and event history](https://docs.temporal.io/workflow-execution) | Adopt | Replay checks deterministic decisions against recorded history and does not mean "run changed code from the middle." | `replay` retains its effect-free meaning; changed code uses `repair`. | Providers, tools, processes, files, and networks cannot dispatch during recorded replay. | Existing replay behavior and JSON contracts remain unchanged. | +| Separate orchestration from nondeterministic work | [Temporal workflow definition and determinism](https://docs.temporal.io/workflow-definition) | Adapt | Temporal places external work in Activities and versions workflow code. `agentctl` already has an effect ledger rather than Temporal Activities. | Graph selection is deterministic; fresh provider/tool/action work is confined to repair roots and descendants. | Reuse is based on persisted results and fingerprints, never rerunning hidden work. | No Temporal history or worker protocol is introduced. | +| Select task boundaries explicitly | [Argo node field selectors](https://argo-workflows.readthedocs.io/en/release-3.7/node-field-selector/) and retry behavior | Adapt | Argo exposes selected-node retry and an explicit successful-node restart option. | `--from` is repeatable and `--restart-successful` is required for a successful repair root. | A successful task cannot be restarted accidentally. | Selection uses stable task IDs in the existing compiled DAG. | +| Invalidate affected descendants | [GitHub Actions rerun jobs API](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2026-03-10) | Adapt | Rerunning a job with its dependent jobs matches the safe default for dataflow invalidation. | Repair executes the union of every root's transitive downstream closure. | No descendant can reuse output derived from a newly executed root. | Independent compatible branches can still be reused. | +| Reuse only reproducible work | [Bazel remote caching](https://bazel.build/remote/caching) | Adopt | Bazel keys reuse from declared inputs and action identity, and verifies content-addressed results. | Each completed task stores versioned definition, resolved-input, contract, output, state-delta, and artifact digests. | Changed inputs, code/configuration, prompt content, output, state, or artifacts block reuse. | Existing runs without repair metadata are readable but cannot be reused automatically. | +| Make cached/reused work visible | [Prefect caching](https://docs.prefect.io/v3/concepts/caching) | Adapt | Prefect combines inputs, code identity, and persisted results, and identifies cached task runs. | Reused tasks remain `succeeded` with `disposition: reused` and source provenance. | Inspection cannot mistake reuse for a fresh attempt. | The existing task state machine remains intact; disposition is additive. | +| Preserve immutable run lineage | [OpenLineage run cycle](https://openlineage.io/docs/spec/run-cycle/) | Adapt | Run events use stable run/job identity and explicit lineage rather than rewriting history. | A repair has a new run/trace ID plus source run ID, source workflow digest, roots, reason, and per-task source attempt. | The terminal source is never reopened or modified. | `parentRunId` retains replay/fork meaning; repair uses explicit source fields. | +| Create repair state atomically | [SQLite transactions](https://www.sqlite.org/lang_transaction.html) | Adopt | An immediate transaction either creates the run, all task materializations, audit events, and checkpoint, or none of them. | Schema migration 5 adds repair lineage and task reuse metadata; repair creation uses one transaction. | Planning/materialization failure cannot leave a runnable partial repair. | Databases migrate forward; unknown newer versions still fail explicitly. | +| Start a repaired agent cleanly | [OpenAI conversation state](https://developers.openai.com/api/docs/guides/conversation-state) | Adopt | `previous_response_id` continues one task-local Responses conversation; it is not a cross-run dataflow mechanism. | A repaired task starts a new provider session. Continuation is used only between turns of that new task. | Failed response IDs, pending tool calls, and uncommitted reasoning are never copied. | Provider continuation within ordinary tasks remains unchanged. | +| Preserve tool-call correlation | [OpenAI Responses migration guidance](https://developers.openai.com/api/docs/guides/migrate-to-responses#additional-differences) | Adopt | Function-call output must correlate to the returned call ID, and stateful continuation has explicit rules. | Repair uses the existing strict tool schema, call-ID, response-ID, and bounded-turn implementation. | Reused upstream data flows through validated JSON output, not hidden conversation state. | OpenAI tool-using agents still require stored continuation in v1alpha1. | +| Keep artifacts trustworthy | [Bazel remote cache protocol](https://bazel.build/remote/caching#remote-caching) | Adapt | Content-addressed output metadata is useful even without adopting a remote cache. | Successful workspace mutations record bounded path, size, and SHA-256 metadata; planning re-resolves the path and verifies size/digest. | Missing, changed, oversized, or path-escaping artifacts block reuse. | Artifact bytes remain in the configured durable workspace; no remote CAS is added. | + +## Patterns rejected + +| Rejected pattern | Reason | +| --- | --- | +| Reopen the terminal source run | It destroys the immutable audit boundary and confuses resume with repair. | +| Treat recorded replay as selective re-execution | It would silently change replay from zero effects to fresh effects. | +| Reuse by task ID alone | IDs do not prove compatible definition, inputs, state, contracts, or artifacts. | +| Initialize from the source final memory snapshot | It can include failed-task or invalidated downstream state. | +| Copy provider continuation across the repair boundary | It carries failed task-local conversation and tool state into a new definition. | +| Generic `--force` | It would turn digest, contract, and effect-uncertainty failures into duplicate-work risks. | +| Claim exactly-once external mutation | A crash can occur after remote commit and before local acknowledgement. | +| Clone another engine's event model | `agentctl` already has a compiled DAG, explicit effects, SQLite checkpoints, and task outputs; a second execution model would add ambiguity. | + +## Resulting invariant + +A task is reusable only if its successful source result, target definition, resolved input boundary, output contract, output bytes, state delta, artifacts, effect certainty, and metadata version all agree. Otherwise planning blocks before a repair run is created and recommends an earlier or additional root. diff --git a/examples/README.md b/examples/README.md index 1a5399e..2f02c33 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,5 @@ # Examples -The supported Rust workflow API examples are in [`v1/`](v1/README.md). The documentation acceptance journeys are in [`docs/`](docs/README.md). +The supported Rust workflow API examples are in [`v1/`](v1/README.md). The documentation acceptance journeys are in [`docs/`](docs/README.md). The two-agent live selective-repair journey is in [`selective-repair-openai/`](selective-repair-openai/README.md). -Other example directories at this level describe the retired TypeScript playbook format and remain only as migration fixtures. They are not accepted by the current `agentctl.dev/v1alpha1` compiler. Do not use their commands as current CLI guidance; use the [migration guide](../docs/MIGRATION.md) to translate them. +Other example directories at this level describe the retired TypeScript playbook format and remain only as migration fixtures. They are not accepted by the current `agentctl.dev/v1alpha1` compiler. Do not use their commands as current CLI guidance; use the [migration guide](../docs/MIGRATING_FROM_TYPESCRIPT.md) to translate them. From af9b4aeec3612474d8d94326164b11a45ee8d14a Mon Sep 17 00:00:00 2001 From: Ompragash Date: Thu, 23 Jul 2026 23:30:31 +0530 Subject: [PATCH 04/44] fix: harden selective repair reuse checks --- crates/agentctl-cli/src/main.rs | 24 +- crates/agentctl-runtime/src/lib.rs | 652 ++++++++++++++++++++++-- docs/LIMITATIONS.md | 2 +- docs/guides/repair-a-failed-workflow.md | 5 +- xtask/src/acceptance.rs | 22 + 5 files changed, 665 insertions(+), 40 deletions(-) diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index f704ebc..a19bb39 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -18,7 +18,7 @@ use agentctl_providers::{ AnthropicProvider, FakeProvider, GoogleProvider, HttpProviderConfig, OpenAiProvider, }; use agentctl_runtime::{BuiltinToolExecutor, RunOptions, Runtime, RuntimeRegistry}; -use agentctl_store::{ApprovalResolution, SqliteStore, StoreError}; +use agentctl_store::{ApprovalResolution, RunMode, SqliteStore, StoreError, TaskDisposition}; use chrono::{Duration as ChronoDuration, Utc}; use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum}; use clap_complete::Shell; @@ -618,7 +618,7 @@ async fn execute(cli: Cli) -> Result { let traces = store .trace_events(&args.run_id) .map_err(CliError::persistence)?; - let human = format!( + let summary = format!( "{} {:?}; {} tasks; {} effects; {} checkpoints; {} audit events; {} traces", args.run_id, run.state, @@ -628,6 +628,26 @@ async fn execute(cli: Cli) -> Result { audit.len(), traces.len(), ); + let human = if run.mode == RunMode::Repair { + let reused = tasks + .iter() + .filter(|task| task.disposition == TaskDisposition::Reused) + .map(|task| task.task_id.as_str()) + .collect::>() + .join(","); + let executed = tasks + .iter() + .filter(|task| task.disposition == TaskDisposition::Executed) + .map(|task| task.task_id.as_str()) + .collect::>() + .join(","); + format!( + "{summary}; source={}; reused={reused}; executed={executed}", + run.source_run_id.as_deref().unwrap_or("unknown") + ) + } else { + summary + }; let value = serde_json::json!({ "run": run, "tasks": tasks, diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index e110d7b..a0ce389 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -978,11 +978,17 @@ impl Runtime { )); continue; } - if source_task.definition_fingerprint.as_deref() != Some(&target_fingerprint) { + let source_needs = source + .plan + .tasks + .get(task_id) + .map(|task| task.needs.as_slice()) + .unwrap_or_default(); + if source_needs != target_task.needs { blocked( - "definition_fingerprint_mismatch", + "dependency_set_mismatch", format!( - "task `{task_id}` changed outside the repair closure; choose it as an earlier repair root" + "task `{task_id}` has a different dependency set in the target workflow" ), false, &mut blocks, @@ -996,17 +1002,11 @@ impl Runtime { )); continue; } - let source_needs = source - .plan - .tasks - .get(task_id) - .map(|task| task.needs.as_slice()) - .unwrap_or_default(); - if source_needs != target_task.needs { + if source_task.definition_fingerprint.as_deref() != Some(&target_fingerprint) { blocked( - "dependency_set_mismatch", + "definition_fingerprint_mismatch", format!( - "task `{task_id}` has a different dependency set in the target workflow" + "task `{task_id}` changed outside the repair closure; choose it as an earlier repair root" ), false, &mut blocks, @@ -1020,6 +1020,44 @@ impl Runtime { )); continue; } + match unresolved_reuse_effects(source_task, &source_effects) { + Ok(effect_ids) if !effect_ids.is_empty() => { + blocked( + "unresolved_reused_effect", + format!( + "task `{task_id}` has unresolved source effect(s) {}; reconcile external reality before reusing its result", + effect_ids.join(", ") + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + Err(message) => { + blocked( + "reuse_effect_provenance", + format!("task `{task_id}` has invalid reused-effect provenance: {message}"), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } + Ok(_) => {} + } if matches!(target_task.uses, TaskUse::Agent(_)) && task_output_schema(target_workflow, target_task).is_none() && target_plan @@ -1117,11 +1155,24 @@ impl Runtime { )); continue; } - let state_delta = source_task.state_delta.as_ref().ok_or_else(|| { - RuntimeError::InvalidState(format!( - "successful source task `{task_id}` has no state delta" - )) - })?; + let Some(state_delta) = source_task.state_delta.as_ref() else { + blocked( + "state_delta_missing", + format!( + "successful source task `{task_id}` has no committed state delta; choose it as an earlier repair root" + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + }; let state_delta_digest = versioned_json_digest(state_delta)?; if source_task.state_delta_digest.as_deref() != Some(&state_delta_digest) { blocked( @@ -1139,7 +1190,24 @@ impl Runtime { )); continue; } - apply_state_delta(&mut memory, state_delta)?; + if let Err(error) = apply_state_delta(&mut memory, state_delta) { + blocked( + "state_delta_invalid", + format!( + "state delta for task `{task_id}` cannot reconstruct boundary memory: {error}" + ), + false, + &mut blocks, + &mut blocked_task_ids, + ); + task_plans.push(blocked_task_plan( + task_id, + Some(source_task.state), + source_fingerprint, + target_fingerprint, + )); + continue; + } outputs.insert(task_id.clone(), output.clone()); let source_effect_summary = if source_task.disposition == TaskDisposition::Reused { source_task @@ -3254,6 +3322,53 @@ fn repair_effect_is_unsafe(effect: &EffectRecord) -> bool { ))) } +fn unresolved_reuse_effects( + task: &TaskRecord, + source_effects: &[EffectRecord], +) -> Result, String> { + if task.disposition != TaskDisposition::Reused { + return Ok(source_effects + .iter() + .filter(|effect| { + effect.request.task_id == task.task_id + && matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) + }) + .map(|effect| effect.request.id.clone()) + .collect()); + } + + let summaries = task + .reuse_decision + .as_ref() + .and_then(|decision| decision.get("sourceEffects")) + .and_then(Value::as_array) + .ok_or_else(|| "sourceEffects is missing or is not an array".to_owned())?; + let mut unresolved = Vec::new(); + for summary in summaries { + let effect_id = summary + .get("effectId") + .and_then(Value::as_str) + .ok_or_else(|| "source effect summary has no effectId".to_owned())?; + let status = summary + .get("status") + .and_then(Value::as_str) + .ok_or_else(|| format!("source effect `{effect_id}` has no status"))?; + match status { + "started" | "uncertain" => unresolved.push(effect_id.to_owned()), + "requested" | "waiting_for_approval" | "succeeded" | "failed" | "cancelled" => {} + other => { + return Err(format!( + "source effect `{effect_id}` has unsupported status `{other}`" + )); + } + } + } + Ok(unresolved) +} + fn task_output_schema(workflow: &Workflow, task: &agentctl_core::CompiledTask) -> Option { task.output_schema.clone().or_else(|| match &task.uses { TaskUse::Agent(name) => workflow @@ -3550,19 +3665,31 @@ fn collect_result_paths(value: &Value, paths: &mut BTreeSet) { fn verify_artifacts(policy: &PolicyEngine, artifacts: &[ArtifactRecord]) -> Result<(), String> { for artifact in artifacts { + let restoration = format!( + "restore `{}` with expected digest `{}` and size {} bytes, or select its producer as an earlier repair root", + artifact.path, artifact.digest, artifact.size_bytes + ); let resolved = policy .resolve_read_path(&artifact.path) - .map_err(|error| error.to_string())?; + .map_err(|error| format!("{restoration}: {error}"))?; let metadata = - std::fs::metadata(&resolved).map_err(|error| format!("{}: {error}", artifact.path))?; + std::fs::metadata(&resolved).map_err(|error| format!("{restoration}: {error}"))?; if metadata.len() != artifact.size_bytes { - return Err(format!("{} size mismatch", artifact.path)); + return Err(format!( + "`{}` size mismatch: expected {} bytes, found {}; {restoration}", + artifact.path, + artifact.size_bytes, + metadata.len() + )); } let content = - std::fs::read(&resolved).map_err(|error| format!("{}: {error}", artifact.path))?; + std::fs::read(&resolved).map_err(|error| format!("{restoration}: {error}"))?; let actual = format!("sha256:{}", digest(&content)); if actual != artifact.digest { - return Err(format!("{} digest mismatch", artifact.path)); + return Err(format!( + "`{}` digest mismatch: expected `{}`, found `{actual}`; {restoration}", + artifact.path, artifact.digest + )); } } Ok(()) @@ -4052,6 +4179,55 @@ mod tests { } } + #[derive(Default)] + struct RepairToolCallingProvider(AtomicU64); + + #[async_trait] + impl ModelProvider for RepairToolCallingProvider { + fn name(&self) -> &'static str { + "fake" + } + + async fn complete( + &self, + _request: &ProviderRequest, + _cancellation: &CancellationToken, + ) -> Result { + let call = self.0.fetch_add(1, Ordering::SeqCst); + if call % 2 == 0 { + Ok(ProviderResponse { + response_id: Some(format!("repair-tool-{call}")), + text: String::new(), + tool_calls: vec![ToolCall { + id: format!("repair-call-{call}"), + name: "echo".to_owned(), + input: serde_json::json!({"text": "durable"}), + }], + assistant_content: vec![ContentBlock::ToolCall { + id: format!("repair-call-{call}"), + name: "echo".to_owned(), + input: serde_json::json!({"text": "durable"}), + provider_metadata: None, + }], + continuation: None, + usage: Usage::default(), + finish_reason: FinishReason::ToolCalls, + }) + } else { + let text = r#"{"value":"durable"}"#.to_owned(); + Ok(ProviderResponse { + response_id: Some(format!("repair-final-{call}")), + text: text.clone(), + tool_calls: Vec::new(), + assistant_content: vec![ContentBlock::Text { text }], + continuation: None, + usage: Usage::default(), + finish_reason: FinishReason::Complete, + }) + } + } + } + struct PanicProvider; #[async_trait] @@ -4150,6 +4326,40 @@ mod tests { } } + struct SingleUseRepairTool { + inner: FixtureTool, + calls: AtomicU64, + } + + impl SingleUseRepairTool { + fn new() -> Self { + Self { + inner: FixtureTool::new(false), + calls: AtomicU64::new(0), + } + } + } + + #[async_trait] + impl ToolExecutor for SingleUseRepairTool { + fn contract(&self) -> &ToolContract { + self.inner.contract() + } + + async fn execute( + &self, + input: Value, + cancellation: &CancellationToken, + ) -> Result { + assert_eq!( + self.calls.fetch_add(1, Ordering::SeqCst), + 0, + "reused upstream tool must not execute during repair" + ); + self.inner.execute(input, cancellation).await + } + } + struct PanicTool { contract: ToolContract, } @@ -4177,6 +4387,78 @@ mod tests { (workflow, plan) } + #[test] + fn repair_fingerprints_include_tool_definitions_and_resolved_task_variables() { + let directory = tempdir().expect("tempdir"); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: fingerprint-repair } +spec: + providers: { fake: { kind: fake } } + tools: + echo: + kind: builtin.echo + description: original echo + inputSchema: { type: object, properties: { text: { type: string } }, required: [text], additionalProperties: false } + outputSchema: { type: object, properties: { text: { type: string } }, required: [text], additionalProperties: false } + capability: internal + risk: low + effectClass: pure + idempotency: pure + retrySafe: true + timeoutSeconds: 5 + approval: never + agents: + worker: + provider: fake + model: fake + instructions: echo the selected value + tools: [echo] + tasks: + - id: first + uses: agent:worker + vars: { selected: "${{ memory.selected }}" } + with: { prompt: "${{ vars.selected }}" } +"#; + let changed_tool = source.replace("original echo", "changed echo"); + let (source_workflow, source_plan) = compile_fixture(source); + let (changed_workflow, changed_plan) = compile_fixture(&changed_tool); + let source_policy = + PolicyEngine::new(source_workflow.spec.policy.clone(), directory.path()) + .expect("source policy"); + let changed_policy = + PolicyEngine::new(changed_workflow.spec.policy.clone(), directory.path()) + .expect("changed policy"); + let source_task = &source_plan.tasks["first"]; + let changed_task = &changed_plan.tasks["first"]; + assert_ne!( + task_definition_fingerprint(&source_workflow, source_task, &source_policy, None) + .expect("source fingerprint"), + task_definition_fingerprint(&changed_workflow, changed_task, &changed_policy, None) + .expect("changed fingerprint") + ); + + let inputs = serde_json::Map::new(); + let outputs = BTreeMap::new(); + assert_ne!( + resolved_input_digest( + &inputs, + &serde_json::json!({"selected": "one"}), + &outputs, + source_task + ) + .expect("first input digest"), + resolved_input_digest( + &inputs, + &serde_json::json!({"selected": "two"}), + &outputs, + source_task + ) + .expect("second input digest") + ); + } + fn runtime(store: SqliteStore, base: &Path) -> Runtime { Runtime::new(store, base) .with_clock(Arc::new(FixedClock)) @@ -4264,6 +4546,10 @@ spec: assert!(plan.compatible, "{:?}", plan.blocked_reuse); assert_eq!(plan.reused_tasks, ["first"]); assert_eq!(plan.rerun_tasks, ["second"]); + let source_before = store.load_run(&source_run_id).expect("source before"); + let source_tasks_before = store + .list_tasks(&source_run_id) + .expect("source tasks before"); let invalid_consumer_yaml = repaired_yaml.replace("tasks.first.output.value", "tasks.first.output.missing"); @@ -4350,6 +4636,16 @@ spec: store.load_run(&source_run_id).expect("source").state, RunState::Failed ); + assert_eq!( + store.load_run(&source_run_id).expect("source after"), + source_before + ); + assert_eq!( + store + .list_tasks(&source_run_id) + .expect("source tasks after"), + source_tasks_before + ); let cutoff = DateTime::from_timestamp(1_767_227_400, 0).expect("cutoff"); store.garbage_collect(cutoff).expect("source gc"); assert!(matches!( @@ -4381,6 +4677,12 @@ spec: .expect("replay sessions") .is_empty() ); + assert!( + store + .tool_calls(&replay.run_id) + .expect("replay tool calls") + .is_empty() + ); let replay_source_plan = runtime .plan_repair( &replay.run_id, @@ -4399,6 +4701,122 @@ spec: ); } + #[tokio::test] + async fn repair_reuses_tool_using_upstream_without_provider_or_tool_dispatch() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(RepairToolCallingProvider::default()); + let tool = Arc::new(SingleUseRepairTool::new()); + let runtime = Runtime::new(store.clone(), directory.path()) + .with_clock(Arc::new(FixedClock)) + .with_ids(Arc::new(SequenceIds::default())) + .with_registry( + RuntimeRegistry::default() + .with_provider("fake", provider.clone()) + .with_tool("echo", tool.clone()), + ); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: repair-tool-reuse } +spec: + providers: { fake: { kind: fake } } + tools: + echo: + kind: builtin.echo + description: echo input + inputSchema: { type: object, properties: { text: { type: string } }, required: [text], additionalProperties: false } + outputSchema: { type: object, properties: { text: { type: string } }, required: [text], additionalProperties: false } + capability: internal + risk: low + effectClass: pure + idempotency: pure + retrySafe: true + timeoutSeconds: 5 + approval: never + agents: + first: + provider: fake + model: fake + instructions: call echo, then return structured output + tools: [echo] + maxTurns: 2 + maxToolCalls: 1 + structuredOutput: + type: object + properties: { value: { type: string } } + required: [value] + additionalProperties: false + actions: + assert: { kind: builtin.assert } + tasks: + - id: first + uses: agent:first + with: { prompt: produce durable output } + - id: second + uses: action:assert + needs: [first] + with: { that: false } +"#; + let target_yaml = source_yaml.replace("that: false", "that: true"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + assert_eq!(provider.0.load(Ordering::SeqCst), 2); + assert_eq!(tool.calls.load(Ordering::SeqCst), 1); + + let (target_workflow, target_plan) = compile_fixture(&target_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &target_workflow, + &target_plan, + &["second".to_owned()], + false, + ) + .expect("repair plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + let repair = runtime + .repair( + &target_workflow, + &target_plan, + plan, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair"); + assert_eq!(repair.state, RunState::Succeeded); + assert_eq!(provider.0.load(Ordering::SeqCst), 2); + assert_eq!(tool.calls.load(Ordering::SeqCst), 1); + assert!( + store + .list_effects(&repair.run_id) + .expect("repair effects") + .iter() + .all(|effect| effect.request.task_id != "first") + ); + assert!( + store + .tool_calls(&repair.run_id) + .expect("repair tool calls") + .iter() + .all(|call| call.task_id != "first") + ); + } + #[tokio::test] async fn repair_plan_uses_minimal_branch_closure_and_blocks_changed_upstream() { let directory = tempdir().expect("tempdir"); @@ -4493,6 +4911,25 @@ spec: block.task_id == "prepare" && block.rule == "definition_fingerprint_mismatch" })); + let changed_dependency_yaml = repaired_yaml.replace( + " - id: analyze_a\n uses: action:assign\n needs: [prepare]", + " - id: analyze_a\n uses: action:assign", + ); + let (dependency_workflow, dependency_plan) = compile_fixture(&changed_dependency_yaml); + let dependency_blocked = runtime + .plan_repair( + &source_run_id, + &dependency_workflow, + &dependency_plan, + &["analyze_b".to_owned()], + false, + ) + .expect("dependency plan"); + assert!(!dependency_blocked.compatible); + assert!(dependency_blocked.blocked_reuse.iter().any(|block| { + block.task_id == "analyze_a" && block.rule == "dependency_set_mismatch" + })); + let changed_contract_yaml = repaired_yaml.replace( " with: { value: stable }", concat!( @@ -4658,6 +5095,91 @@ spec: assert_eq!(compatible.fresh_effect_summary.uncertain_source_effects, 1); } + #[tokio::test] + async fn repair_rejects_unresolved_effect_on_otherwise_reusable_task() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: reused-effect-repair } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - { id: first, uses: "action:assign", with: { value: durable } } + - { id: second, uses: "action:assert", needs: [first], with: { that: false } } +"#; + let repaired_yaml = source_yaml.replace("that: false", "that: true"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let unresolved = EffectRequest::new( + &source_run_id, + "first", + 1, + 99, + "external.ambiguous", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"record": "x"}), + "create external record", + "trace-reused-uncertain", + ); + store + .record_effect_request(&unresolved, FixedClock.now()) + .expect("record effect"); + store + .mark_effect_started(&unresolved.id, FixedClock.now()) + .expect("start effect"); + + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let stats_before = store.stats().expect("stats"); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("blocked plan"); + assert!(!plan.compatible); + assert!(plan.blocked_reuse.iter().any(|block| { + block.task_id == "first" + && block.rule == "unresolved_reused_effect" + && block.message.contains(&unresolved.id) + })); + assert!(matches!( + runtime + .repair( + &repaired_workflow, + &repaired_plan, + plan, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await, + Err(RuntimeError::RepairBlocked { .. }) + )); + assert_eq!(store.stats().expect("stats"), stats_before); + } + #[tokio::test] async fn repair_detects_changed_upstream_prompt_file() { let directory = tempdir().expect("tempdir"); @@ -4806,7 +5328,9 @@ spec: async fn repair_blocks_when_reused_artifact_is_missing() { let directory = tempdir().expect("tempdir"); let store = SqliteStore::open_memory().expect("store"); - let runtime = runtime(store.clone(), directory.path()); + let runtime = runtime(store.clone(), directory.path()).with_registry( + RuntimeRegistry::default().with_provider("fake", Arc::new(PanicProvider)), + ); let source_yaml = r#" apiVersion: agentctl.dev/v1alpha1 kind: Workflow @@ -4828,7 +5352,24 @@ spec: needs: [first] with: { that: false } "#; - let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let repaired_yaml = source_yaml + .replace( + "spec:\n", + concat!( + "spec:\n", + " providers:\n", + " fake: { kind: fake }\n", + " agents:\n", + " repaired:\n", + " provider: fake\n", + " model: fake\n", + " instructions: must never execute when the artifact is missing\n" + ), + ) + .replace( + " - id: second\n uses: action:assert\n needs: [first]\n with: { that: false }", + " - id: second\n uses: agent:repaired\n needs: [first]\n with: { prompt: repaired }", + ); let (source_workflow, source_plan) = compile_fixture(source_yaml); let source_run_id = match runtime .start( @@ -4854,8 +5395,11 @@ spec: ) .expect("plan"); assert!(compatible.compatible, "{:?}", compatible.blocked_reuse); + let expected_digest = compatible.materialized_tasks[0].metadata.artifact_manifest[0] + .digest + .clone(); std::fs::remove_file(directory.path().join("artifact.txt")).expect("remove artifact"); - let runs_before = store.stats().expect("stats").runs; + let stats_before = store.stats().expect("stats"); assert!(matches!( runtime .repair( @@ -4869,7 +5413,7 @@ spec: .await, Err(RuntimeError::RepairBlocked { .. }) )); - assert_eq!(store.stats().expect("stats").runs, runs_before); + assert_eq!(store.stats().expect("stats"), stats_before); let plan = runtime .plan_repair( &source_run_id, @@ -4880,15 +5424,18 @@ spec: ) .expect("blocked plan"); assert!(!plan.compatible); - assert!( - plan.blocked_reuse - .iter() - .any(|block| { block.task_id == "first" && block.rule == "artifact_integrity" }) - ); + assert!(plan.blocked_reuse.iter().any(|block| { + block.task_id == "first" + && block.rule == "artifact_integrity" + && block.message.contains("artifact.txt") + && block.message.contains(&expected_digest) + && block.message.contains("earlier repair root") + })); + assert_eq!(store.stats().expect("stats"), stats_before); } #[tokio::test] - async fn repair_blocks_tampered_reused_output_digest() { + async fn repair_blocks_missing_state_delta_and_tampered_reused_output_digest() { let directory = tempdir().expect("tempdir"); let database = directory.path().join("runtime.db"); let store = SqliteStore::open(&database).expect("store"); @@ -4925,14 +5472,47 @@ spec: Err(RuntimeError::RunFailed { run_id, .. }) => run_id, other => panic!("expected source failure, got {other:?}"), }; - rusqlite::Connection::open(&database) - .expect("tamper connection") + let tamper = rusqlite::Connection::open(&database).expect("tamper connection"); + tamper + .execute( + "UPDATE task_states SET state_delta_json = NULL WHERE run_id = ?1 AND task_id = ?2", + rusqlite::params![source_run_id, "first"], + ) + .expect("remove state delta"); + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let missing_delta = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("missing state-delta plan"); + assert!(!missing_delta.compatible); + assert!( + missing_delta + .blocked_reuse + .iter() + .any(|block| { block.task_id == "first" && block.rule == "state_delta_missing" }) + ); + + tamper + .execute( + "UPDATE task_states SET state_delta_json = ?3 WHERE run_id = ?1 AND task_id = ?2", + rusqlite::params![ + source_run_id, + "first", + r#"{"formatVersion":1,"set":{},"remove":[]}"# + ], + ) + .expect("restore state delta"); + tamper .execute( "UPDATE task_states SET output_json = ?3 WHERE run_id = ?1 AND task_id = ?2", rusqlite::params![source_run_id, "first", r#"{"tampered":true}"#], ) .expect("tamper output"); - let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); let plan = runtime .plan_repair( &source_run_id, diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index c7568a1..d28be49 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -45,7 +45,7 @@ These are useful extensions but are not required by the product thesis. They nee - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. - At-most-once model/remote calls can become uncertain in the dispatch/acknowledgement window. Inspect and reconcile externally; use `fork` only when fresh effects are knowingly acceptable. - Selective repair requires task metadata version 1. Successful tasks from databases created before schema 5 remain inspectable but must execute from an earlier repair root or a full fork. -- Automatic artifact manifests cover bounded files reported by successful workspace-mutation results. Artifact bytes are not copied into SQLite or a content-addressed store; retain the configured workspace and restore by verified digest when needed. +- Automatic artifact manifests cover bounded files reported by successful workspace-mutation results. Artifact bytes are not copied into SQLite or a content-addressed store; retain the configured workspace. Missing, moved, size-mismatched, or digest-mismatched bytes block repair before run creation and report the expected artifact identity. - A confirmed non-idempotent mutation in a repair closure remains blocked. The only built-in reconciliation outcome is an operator-confirmed `not-applied` result for a started or uncertain effect; compensation and provider-specific deduplication workflows are not implemented. - Retry remains a bounded same-run task policy. There is no separate command that creates a new terminal-source retry run for an unchanged workflow; use repair with an unchanged target definition and explicit roots when its compatibility checks fit. - Tool-using OpenAI/Azure agents require stored-response continuation. `store: false` is rejected until stateless response-item replay is implemented. diff --git a/docs/guides/repair-a-failed-workflow.md b/docs/guides/repair-a-failed-workflow.md index 5f74f10..7749c5a 100644 --- a/docs/guides/repair-a-failed-workflow.md +++ b/docs/guides/repair-a-failed-workflow.md @@ -176,11 +176,14 @@ Recorded replay has a new replay run ID but the same semantic outputs. It dispat | `repair_root_missing` | The root is absent from the target graph. | Correct the task ID or workflow. | | `successful_root_requires_acknowledgement` | The selected root succeeded. | Add `--restart-successful` only when fresh execution is intended. | | `definition_fingerprint_mismatch` | A task changed outside the rerun closure. | Choose that task as an earlier/additional root. | +| `dependency_set_mismatch` | A reusable task has different upstream dependencies. | Select the changed consumer as another repair root. | | `resolved_input_digest_mismatch` | Inputs, dependency output, or boundary memory changed. | Choose the first affected task as a root. | | `missing_output_contract` | A reused agent feeds downstream work without typed output. | Add structured output and create a fresh source result. | | `output_contract_mismatch` | The target expects a different contract. | Rerun from the producer. | | `output_digest_mismatch` | Persisted output was modified or corrupted. | Do not reuse it; rerun from the producer. | -| `artifact_integrity` | An artifact is missing, changed, or outside policy. | Restore the verified artifact or rerun its producer. | +| `state_delta_missing` or `state_delta_invalid` | Successful boundary-state metadata is absent or corrupt. | Select the task as an earlier root; do not edit the database. | +| `artifact_integrity` | An artifact is missing, changed, or outside policy. The block reports its path, expected digest, and expected size. | Restore the exact retained artifact or select its producer as an earlier repair root. | +| `unresolved_reused_effect` | A nominally successful reusable task retains a started or uncertain effect. | Reconcile external reality before reuse. | | `legacy_task_metadata` | The source predates repair metadata v1. | Use an earlier root or a full fork. | | `new_task_outside_repair_closure` | A new unrelated task has no result. | Add it as a root or choose an earlier common boundary. | | `unreconciled_effect` | Fresh execution may duplicate a mutation. | Inspect and reconcile external reality first. | diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 9b52cd0..8e7a36c 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -778,6 +778,28 @@ pub fn run(root: &Path) -> Result<()> { ensure_eq(&repair_inspect, "/data/tasks/0/disposition", "reused")?; ensure_eq(&repair_inspect, "/data/tasks/1/disposition", "executed")?; ensure!(array_len(&repair_inspect, "/data/effects")? == 0); + let human_inspect = output_with_code( + command_for( + &binary, + &workspace, + &strings([ + "inspect", + repair_run_id, + "--db", + path(&repair_db)?, + "--output", + "human", + "--color", + "never", + ]), + ), + 0, + "human repair inspection", + )?; + let human_inspect = String::from_utf8_lossy(&human_inspect.stdout); + ensure!(human_inspect.contains(&format!("source={source_run_id}"))); + ensure!(human_inspect.contains("reused=first")); + ensure!(human_inspect.contains("executed=second,third")); scenario( 27, From 841b1b53c1d3c23314ec914aef124b1ecada07c5 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 15:24:47 +0530 Subject: [PATCH 05/44] docs: define framework completeness program --- docs/execution/COMPLETENESS_VERIFICATION.md | 92 +++ docs/execution/FRAMEWORK_COMPLETENESS.md | 114 ++++ docs/execution/LIMITATION_BURNDOWN.md | 674 ++++++++++++++++++++ 3 files changed, 880 insertions(+) create mode 100644 docs/execution/COMPLETENESS_VERIFICATION.md create mode 100644 docs/execution/FRAMEWORK_COMPLETENESS.md create mode 100644 docs/execution/LIMITATION_BURNDOWN.md diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md new file mode 100644 index 0000000..4483fe5 --- /dev/null +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -0,0 +1,92 @@ +# Framework completeness verification + +This record accumulates sanitized evidence for the framework-completeness +program. It contains no credentials, raw provider responses, prompt +transcripts, runtime databases, private certificates, or artifact bytes. + +## Baseline + +Branch point: `af9b4ae`, including independently reviewed selective workflow +repair. + +Date: 2026-07-23, Asia/Kolkata. + +| Gate | Baseline result | +| --- | --- | +| `cargo xtask verify` | passed all 12 stages | +| `cargo xtask acceptance` | passed 28 scenarios | +| `cargo xtask examples-verify` | passed | +| `cargo xtask docs-verify` | passed all 6 stages | +| `cargo xtask package` | passed for macOS arm64 | +| `cargo xtask secret-scan` | passed | +| `env -u OPENAI_API_KEY cargo xtask acceptance-container` | reached the OCI CLI, then failed because `/artifacts/report.txt` escaped the authorized workspace root | + +The installed container runtime is Podman 5.8.2 with a libkrun machine. In this +execution environment, Podman's VM and forwarding processes survive only while +the starting terminal remains open. Keeping that terminal active made the +engine reachable without deleting a machine, changing TLS, or weakening +configuration. + +No OpenAI request was made during baseline verification. + +## Evidence rules + +- Deterministic gates run without provider credentials. +- Live gates use only `gpt-5.6`, at most 80 Responses API requests for this + program, and a target aggregate cost below USD 15. +- Live records retain only scenario, model, request/tool counts, token counts, + run ID, outcome, and recovery/replay reuse status. +- Raw model content, databases, and keys stay in ignored local evidence. +- Configured hosted jobs are not described as executed. +- Native and emulated container architecture results are labeled explicitly. +- Every verified limitation links to focused tests plus at least one public + product path. + +## Required deterministic gates + +```console +cargo xtask verify +cargo xtask acceptance +cargo xtask examples-verify +cargo xtask docs-verify +cargo xtask package +cargo xtask secret-scan +cargo xtask artifact-store-verify +cargo xtask migration-verify +cargo xtask protocol-resilience +cargo xtask completeness +``` + +## Required opt-in gates + +```console +cargo xtask examples-verify-live-openai +cargo xtask acceptance-container +``` + +## Workstream evidence + +| Workstream | Focused evidence | Composite evidence | Status | +| --- | --- | --- | --- | +| Artifact CAS | pending | pending | open | +| Legacy upgrades | pending | pending | open | +| Reconciliation | pending | pending | open | +| Terminal retry | pending | pending | open | +| Parallel/dynamic workflows | pending | pending | open | +| Conditions/loops/sub-workflows | pending | pending | open | +| Compensation/handoffs/streaming | pending | pending | open | +| MCP/A2A resilience | pending | pending | open | +| Packs/trust/extensions | pending | pending | open | +| Semantic memory | pending | pending | open | +| Encryption/secrets | pending | pending | open | +| Network/isolation/budgets | pending | pending | open | +| Container/cross-platform | baseline defect recorded | pending | in progress | +| OpenAI live matrix | retained selective-repair evidence only | pending | open | +| Canonical and Pages docs | limitation register created | pending | in progress | + +## Final adversarial review + +No final review result is recorded yet. Completion requires a clean review pass +over security, migrations, effects, artifacts, concurrency, recovery, +protocols, pack trust, encryption, budgets, containers, examples, and live +evidence, followed by remediation of every P0/P1 and scoped P2 finding. diff --git a/docs/execution/FRAMEWORK_COMPLETENESS.md b/docs/execution/FRAMEWORK_COMPLETENESS.md new file mode 100644 index 0000000..a0e76c1 --- /dev/null +++ b/docs/execution/FRAMEWORK_COMPLETENESS.md @@ -0,0 +1,114 @@ +# Complete framework product surface + +## Product contract + +`agentctl` is a deterministic and declarative runtime for durable agentic +workflows. The compiler, graph, state machine, effect ledger, policy, and +versioned persistence remain authoritative. Models execute bounded typed tasks; +they do not own orchestration. + +The complete supported surface contains: + +- strict versioned YAML, typed inputs/outputs, constrained templates, typed + conditions, and deterministic routing; +- sequential or bounded parallel DAG execution with stable commit order and + explicit working-memory conflict rules; +- bounded foreach/matrix expansion, bounded loops, and namespaced + sub-workflows; +- deterministic actions, bounded provider tasks, typed tools, structured + handoffs, explicit compensation, and durable streaming events; +- local SQLite history, authenticated encryption for selected sensitive fields, + content-addressed artifacts, checkpoints, audit, traces, and usage budgets; +- resume, operator reconciliation, terminal retry, selective repair, recorded + replay, and explicit effectful fork; +- native provider adapters behind a provider-neutral interface; +- resilient MCP and A2A clients that never duplicate uncertain mutation; +- local/Git/immutable-archive packs with deterministic locking and optional + established signature verification; +- reviewed packs, MCP, and a bounded process protocol as the extension model; +- exact, text, vector, and hybrid optional long-term-memory retrieval with + explicit promotion; +- environment, mounted-file, and policy-gated process secret references whose + values are never persisted; +- explicit network and process policies, honest isolation modes, and generic + non-root/read-only container execution; +- human, final JSON, JSONL progress, inspection, export, migration, and + administration commands; +- local, externally scheduled, CI, container, and embedded Rust operation. + +## Determinism rules + +1. Compilation fixes task identity, dependencies, expansion limits, schemas, + policies, budgets, and commit order. +2. Parallel tasks read an immutable boundary snapshot. Their durable commits + occur in compiled order. +3. Working-memory writes are declared. Conflicts fail before effect dispatch + unless a versioned deterministic merge is explicit. +4. Dynamic children and loop iterations receive stable IDs and hard bounds. +5. Every external observation or mutation has a persisted identity before + dispatch. +6. Uncertain mutating work is never silently repeated. +7. Provider sessions are task-local. Typed output, durable state, artifacts, + and explicit handoff payloads are the only cross-task dataflow. +8. Recorded replay dispatches no provider, tool, process, network, filesystem, + protocol, memory, or artifact-ingestion effect. + +## Persistence boundaries + +SQLite stores versioned metadata and encrypted sensitive fields. Artifact bytes +live in an immutable local content-addressed store rooted beside the database. +The database references blobs by digest and owns reachability and retention. +External artifact backends may implement the same interface later, but none is +required for the complete local product. + +The database and artifact root are backed up together. A repair or retry run +materializes its own metadata references so source-row garbage collection does +not break it. Blob garbage collection removes only unreferenced content outside +the configured retention window. + +## Recovery operations + +- `resume` continues a nonterminal run and reuses confirmed effects. +- `reconcile` records operator-confirmed external reality without mutating the + source effect. +- `retry` creates a new source-linked run for an identical workflow and reruns + failed or explicitly selected boundaries. +- `repair` creates a new source-linked run for a changed compatible workflow. +- `replay` reconstructs a terminal run and recorded stream without fresh + effects. +- `fork` creates a broad new execution with intentionally fresh effects. +- `compensate` performs explicitly declared best-effort reverse actions and + never claims transactional rollback. + +## Extension contract + +There is no in-process native plugin ABI. Extensions use one of: + +- reviewed declarative packs; +- MCP for remote tools; +- a versioned bounded process protocol for local executors. + +Each executable extension declares schemas, capabilities, effects, limits, and +policy requirements. Process and MCP execution remain isolated effect +boundaries. + +## Explicit non-goals + +- hosted SaaS, public control plane, or public pack registry; +- chat application, free-form multi-agent conversation, or hidden model-owned + routing; +- Kubernetes operator, runner fleet, public cloud scheduler, calendars, or + event triggers; +- distributed scheduling, distributed leases, multi-host execution, or + distributed storage; +- IDE or visual workflow editor; +- general configuration management; +- unbounded loops, unbounded model-controlled expansion, or arbitrary + expression code; +- an unsafe in-process native plugin ABI; +- a claim that policy allowlists are an OS sandbox; +- a claim of exactly-once external mutation or transactional compensation. + +External schedulers own triggers and overlap policy. Containers, VMs, platform +identities, and egress controls remain the strongest isolation boundary for +hostile workloads. diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md new file mode 100644 index 0000000..aa1199d --- /dev/null +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -0,0 +1,674 @@ +# Framework limitation burn-down + +This is the authoritative register for the framework-completeness program. It +supersedes roadmap language that classified core durability, recovery, +orchestration, security, or operability work as deferred merely because the +workflow API is young. + +Program state values are `open`, `in progress`, and `verified`. They describe +the active work queue and are not final dispositions. Before this program is +complete, every entry must have exactly one final disposition: + +- `implemented` +- `redesigned` +- `removed from supported surface` +- `externally blocked` + +## Dependency order + +1. Persistence foundations: artifact content addressing, schema migration, + sensitive-field encryption, and durable reconciliation. +2. Recovery contracts: legacy-run analysis, terminal retry, compensation, and + artifact-independent repair/replay. +3. Deterministic scheduler: parallel commits, conflict detection, bounded + expansion, conditions, loops, and sub-workflows. +4. Bounded agent composition and event output: structured handoffs and + streaming. +5. Remote and extension boundaries: MCP, A2A, pack locking, trust, and the + isolated extension protocol. +6. Optional semantic memory, network/process isolation, and resource budgets. +7. Composite acceptance, container/cross-platform evidence, live OpenAI proof, + documentation, and adversarial review. + +## Register summary + +| ID | Category | Program state | Intended final disposition | +| --- | --- | --- | --- | +| ART-001 | Durable artifacts | verified | implemented | +| MIG-001 | Legacy selective repair | open | implemented | +| EFX-001 | Effect reconciliation | open | implemented | +| RET-001 | Terminal-run retry | open | implemented | +| ENC-001 | Sensitive-state encryption | open | implemented | +| SEC-001 | Secret providers | open | implemented | +| NET-001 | Network policy | open | implemented | +| ISO-001 | Process isolation | open | redesigned | +| BUD-001 | Resource and cost budgets | open | implemented | +| SCH-001 | Deterministic parallel execution | open | implemented | +| DYN-001 | Foreach and matrix | open | implemented | +| COND-001 | Conditions and routers | open | implemented | +| LOOP-001 | Bounded loops | open | implemented | +| SUB-001 | Sub-workflows | open | implemented | +| COMP-001 | Compensation | open | implemented | +| TEAM-001 | Structured teams and handoffs | open | redesigned | +| STR-001 | Streaming | open | implemented | +| MCP-001 | MCP resilience | open | implemented | +| A2A-001 | A2A resilience | open | implemented | +| PACK-001 | Pack resolution and lockfiles | open | implemented | +| TRUST-001 | Pack integrity and signing | open | implemented | +| EXT-001 | Plugin strategy | open | redesigned | +| MEM-001 | Semantic memory | open | implemented | +| PROV-001 | Stateless provider continuation | open | implemented | +| OCI-001 | Container execution | in progress | implemented | +| XPLAT-001 | Cross-platform hosted evidence | open | externally blocked | +| EVENT-001 | Event triggers and calendars | verified | removed from supported surface | +| DIST-001 | Distributed execution and storage | verified | removed from supported surface | +| REG-001 | Hosted public registry | verified | removed from supported surface | +| UI-001 | Hosted UI, chat, and visual orchestration | verified | removed from supported surface | + +## Persistence and recovery + +### ART-001: Durable content-addressed artifacts + +- Current behavior: successful bounded file outputs are atomically ingested + into an immutable local SHA-256 CAS beside the database. SQLite stores blob + metadata, per-run/task references, provenance, and ingestion leases. +- User impact: repair, replay, verification, and export continue after the + source workspace file is deleted. +- Security or durability impact: bytes and metadata form one backup boundary; + digest verification detects missing/corrupt blobs. +- Product decision: use a local filesystem content-addressed store beside the + state database. SQLite owns metadata, references, provenance, retention, and + reachability. Blob bytes never enter ordinary SQLite rows. +- Required implementation: atomic verified ingestion, immutable deduplicated + blobs, media type and logical-name metadata, run/task references, corruption + verification, export/materialization, inspection, and reachability GC. +- Migration impact: schema 6 adds CAS metadata/reference/lease tables. Explicit + legacy analysis/import is tracked separately by MIG-001. +- Tests: duplicate ingestion, partial writes, corrupt/missing/wrong-digest + blobs, disk failures, concurrent ingestion, traversal/symlink rejection, + workspace/source deletion, repair, replay, GC, read-only consumption, and + redaction. +- Examples: durable pipeline and container pipeline. +- Live evidence: bounded OpenAI artifact-producing repair plus offline replay. +- Documentation: artifact store, container mounts, backup, repair, replay, and + GC. +- Final disposition: implemented and verified by 19 store tests, 38 runtime + tests, credential-free artifact CLI acceptance, and hardened OCI acceptance. + +### MIG-001: Legacy run analysis and upgrade + +- Current behavior: schema-v5 migration preserves old tasks but cannot safely + reuse tasks that lack repair metadata version 1. +- User impact: operators must choose an unnecessarily broad root or full fork. +- Security or durability impact: fabricating missing fingerprints or deltas + would permit unsafe reuse. +- Product decision: implement transactional dry-run analysis and an explicit + run upgrade. Derive only provable metadata and calculate the earliest safe + root for everything else. +- Required implementation: `runs upgrade` analysis/apply UX, confidence and + provenance records, digest derivation, checkpoint-delta reconstruction, + artifact import, and earliest-safe-boundary output. +- Migration impact: every retained schema fixture remains readable; upgrades + are additive and source records remain immutable. +- Tests: schema fixtures 1 through 5, complete/partial/impossible derivation, + failed-upgrade rollback, dry run, corrupt checkpoints, and boundary choice. +- Examples: legacy analysis followed by selective retry/repair. +- Live evidence: not required; the contract is deterministic. +- Documentation: database migration, compatibility, and operator guidance. +- Final disposition: pending implementation evidence. + +### EFX-001: Complete operator reconciliation + +- Current behavior: only a started or uncertain effect can be changed to a + failed `not_applied` state. +- User impact: applied, compensated, or externally completed work cannot be + represented safely. +- Security or durability impact: operators may resort to unsafe forks or + out-of-band database edits. +- Product decision: preserve immutable source effects and append versioned + reconciliation records with one active conclusion. +- Required implementation: list, inspect, and reconcile outcomes `applied`, + `not_applied`, and `compensated`; identity, timestamp, reason, evidence, + optional validated result, supersession rules, compensation linkage, audit, + trace, policy, and non-interactive behavior. +- Migration impact: new reconciliation table and effective-effect projection; + existing `not_applied` audits migrate to records when provable. +- Tests: every transition, contradictory decisions, supersession, wrong + schemas, operator policy, repair/resume/retry integration, idempotency keys, + and transaction rollback. +- Examples: operational workflow with manual applied and compensated outcomes. +- Live evidence: selective repair after an explicitly reconciled mock effect. +- Documentation: effect recovery and honest external-state semantics. +- Final disposition: pending implementation evidence. + +### RET-001: Terminal-run retry + +- Current behavior: task retry is same-run and bounded; terminal rerun requires + repair or a broad fork. +- User impact: operational retry of a failed unchanged workflow is obscure. +- Security or durability impact: a fork can repeat successful external effects. +- Product decision: add a distinct source-linked retry plan and run mode. It + requires the identical workflow definition and reuses compatible success. +- Required implementation: failed-only, selected roots, multiple roots, + successful-root acknowledgement, plan/JSON/human output, fresh attempts, + lineage, effect safety, reconciliation, and offline replay. +- Migration impact: new run mode and source/roots metadata are additive. +- Tests: failed-only closure, branches, multiple roots, workflow mismatch, + explicit successful restart, uncertain effects, source immutability, and + replay. +- Examples: durable pipeline retry after deterministic downstream failure. +- Live evidence: bounded deterministic provider failure followed by live retry. +- Documentation: retry versus resume, repair, replay, and fork. +- Final disposition: pending implementation evidence. + +### ENC-001: Envelope encryption for sensitive persisted fields + +- Current behavior: SQLite may contain prompts, inputs, outputs, tool data, and + provider continuations in plaintext. +- User impact: filesystem disclosure reveals confidential workflow history. +- Security or durability impact: SQLite permissions are not confidentiality at + rest. +- Product decision: use an established authenticated-encryption crate and a + versioned envelope for identified sensitive JSON/text fields. Do not claim + full-database encryption. +- Required implementation: key references, key IDs, authenticated associated + data, strict no-fallback decryption, rotation, redacted inspection, + backup/restore guidance, and bounded migration. +- Migration impact: transactional plaintext-to-envelope migration with dry-run + inventory and rollback on wrong/missing keys. +- Tests: known vectors where provided by the library, wrong key, tampering, + rotation, mixed versions, migration rollback, and no plaintext remnants in + protected columns. +- Examples: encrypted state with redacted inspection. +- Live evidence: no provider call required. +- Documentation: protected fields, key lifecycle, and residual metadata. +- Final disposition: pending implementation evidence. + +## Workflow language and runtime + +### SCH-001: Deterministic parallel scheduling + +- Current behavior: `maxConcurrency` must be 1. +- User impact: independent model and deterministic tasks cannot overlap. +- Security or durability impact: naive concurrency would make state and effect + order race-dependent. +- Product decision: execute a stable ready batch concurrently but commit task + results in compiled order. Tasks declare working-memory write sets; conflicting + writes fail before dispatch unless an explicit deterministic merge exists. +- Required implementation: concurrency semaphore, isolated task snapshots, + ordered commit queue, failure/cancellation/approval behavior, effect and trace + parentage, repair/retry/replay integration, and plan visibility. +- Migration impact: runtime/plan format versions and DSL schema change. +- Tests: real overlap, stable commit order, conflict rejection, cancellation, + approval, failures, effects, replay, repair, and container execution. +- Examples: parallel deterministic and agent branches. +- Live evidence: two bounded OpenAI branches. +- Documentation: scheduling and state conflict rules. +- Final disposition: pending implementation evidence. + +### DYN-001: Bounded foreach and matrix expansion + +- Current behavior: no dynamic task expansion. +- User impact: authors duplicate similar tasks and cannot retry individual + expanded units. +- Security or durability impact: model-controlled unbounded expansion could + exhaust resources. +- Product decision: compile static matrices and deterministically expand + runtime arrays only from typed non-model inputs or validated bounded outputs. +- Required implementation: stable escaped child IDs, item binding, count + limits, aggregate output, partial-failure rules, child inspection, + repair/retry/replay, and task budgets. +- Migration impact: plan/checkpoint formats gain expansion records. +- Tests: order, ID collisions, bounds, aggregation, partial failure, individual + retry/repair, replay, and malformed input. +- Examples: small deterministic and agent matrices. +- Live evidence: two-item OpenAI matrix. +- Documentation: syntax, limits, IDs, and recovery. +- Final disposition: pending implementation evidence. + +### COND-001: Typed conditions and routers + +- Current behavior: constrained `not` and equality conditions exist, but there + is no explicit router or complete skipped-output contract. +- User impact: nontrivial deterministic branching is awkward. +- Security or durability impact: expanding to arbitrary expressions would add + code execution and ambiguous dependencies. +- Product decision: version the existing constrained expression AST and add a + typed route selector with enumerated destinations. +- Required implementation: compile-time validation, durable evaluation input + and decision, skipped-state semantics, changed-input invalidation, plan + visibility, output option contracts, repair/retry/replay. +- Migration impact: task state and plan versions add condition decisions. +- Tests: types, missing/null, invalid routes, skips, downstream behavior, + changed decisions, repair, and replay. +- Examples: structured agent output routed to deterministic branches. +- Live evidence: one structured-output routing scenario. +- Documentation: expression grammar and skip semantics. +- Final disposition: pending implementation evidence. + +### LOOP-001: Bounded loops + +- Current behavior: loops are rejected. +- User impact: bounded refine/verify workflows require duplicated tasks. +- Security or durability impact: an unbounded model-owned loop violates the + runtime's bounded-execution thesis. +- Product decision: implement a durable loop construct with typed condition, + explicit maximum iterations, and iteration-local output/state boundaries. +- Required implementation: stable iteration IDs, durable iteration state, + outputs, cancellation, effect identities, repair/retry at boundaries, replay, + and loop/resource budgets. +- Migration impact: plan, checkpoint, and task-attempt formats. +- Tests: zero/one/max iterations, bound exceeded, cancellation, uncertain + effect, repair/retry, and replay. +- Examples: bounded operational verification loop. +- Live evidence: a two-iteration maximum agent scenario. +- Documentation: loop safety and recovery. +- Final disposition: pending implementation evidence. + +### SUB-001: Reusable sub-workflows + +- Current behavior: packs can contribute actions/agents/tools but not workflows. +- User impact: reusable graph composition requires copying tasks. +- Security or durability impact: implicit policy/provider inheritance could + broaden authority. +- Product decision: compile versioned sub-workflows into namespaced tasks with + explicit typed inputs/outputs and monotonic policy inheritance. +- Required implementation: pack/local definitions, namespace escaping, + recursion/cycle checks, state isolation, provider mapping, artifact ownership, + lineage, errors, inspection, repair/retry/replay. +- Migration impact: pack/lock, workflow schema, and plan format. +- Tests: nesting, collisions, cycles, policy narrowing, output contracts, + failures, artifacts, repair/retry/replay. +- Examples: operational workflow calling a reusable sub-workflow. +- Live evidence: sub-workflow containing one OpenAI task. +- Documentation: authoring, versioning, and policy inheritance. +- Final disposition: pending implementation evidence. + +### COMP-001: Explicit compensation + +- Current behavior: tool contracts carry compensation metadata but runtime does + not execute it. +- User impact: operators cannot durably coordinate best-effort reversal. +- Security or durability impact: documentation-shaped metadata can be mistaken + for transactional rollback. +- Product decision: compensation is an explicit new run phase in reverse + dependency order, never a transactional rollback claim. +- Required implementation: declaration validation, manual trigger, opt-in + automatic trigger, approval, idempotency, partial failure, linkage to effects + and reconciliation, audit, trace, retry/repair behavior. +- Migration impact: effect links, run phase, and checkpoints. +- Tests: order, approval, idempotency, partial failure, contradictory + reconciliation, cancellation, and replay. +- Examples: operational workflow compensation. +- Live evidence: deterministic tool compensation only. +- Documentation: guarantees and non-guarantees. +- Final disposition: pending implementation evidence. + +### TEAM-001: Structured teams and handoffs + +- Current behavior: free-form agent handoffs are intentionally absent. +- User impact: users cannot name bounded roles and inspect typed handoffs. +- Security or durability impact: hidden agent conversations would bypass the + compiled graph and policy. +- Product decision: redesign "teams" as syntactic composition over explicit + tasks/sub-workflows. No autonomous hidden conversation scheduler is added. +- Required implementation: role declarations, bounded turn count, typed handoff + payload, explicit route conditions, per-role tools/policy, durable handoff + records, and ordinary repair/retry/replay. +- Migration impact: new syntax compiles to existing versioned task constructs. +- Tests: policy separation, handoff schemas, turn limits, routes, failures, + repair/retry/replay, and inspection. +- Examples: three-role structured verification workflow. +- Live evidence: two-role OpenAI handoff plus deterministic verifier. +- Documentation: explain the compiled replacement and reject free-form teams. +- Final disposition: pending redesign evidence. + +### STR-001: End-to-end streaming + +- Current behavior: provider transports parse some SSE but workflow output is a + final document only. +- User impact: long agent calls have no bounded progress stream. +- Security or durability impact: unbounded deltas or mixed JSON output can leak + secrets and corrupt automation. +- Product decision: add durable bounded stream events and explicit human or + JSONL progress modes while retaining one final JSON document mode. +- Required implementation: provider fragments, sequence numbers, persisted + bounded/redacted records, backpressure, cancellation, reconnect semantics, + final result validation, and recorded stream replay. +- Migration impact: stream-event table and CLI output contract. +- Tests: fragmented events, backpressure, truncation, redaction, cancellation, + replay, reconnect, and final JSON isolation. +- Examples: streaming agent workflow. +- Live evidence: one packaged OpenAI streaming run. +- Documentation: stdout contracts and replay. +- Final disposition: pending implementation evidence. + +## Remote protocols, packs, and memory + +### MCP-001: Safe MCP reconnect + +- Current behavior: session expiry fails explicitly and requires external + recovery. +- User impact: safe observations cannot recover from server restart, and manual + recovery lacks protocol-specific evidence. +- Security or durability impact: automatic retry of an uncertain mutation can + duplicate work. +- Product decision: bounded reconnect is allowed before dispatch and for proven + observations/idempotent calls. Uncertain mutating calls require EFX-001. +- Required implementation: reinitialize, tool-list/schema refresh, auth refresh, + server restart handling, reconnect budget, call identity, streaming, and + repair/retry/replay integration. +- Migration impact: protocol session and call records gain generation/status. +- Tests: restart at each lifecycle boundary, schema change, auth refresh, + cancellation, timeout, and no duplicate mutation. +- Examples: operational mock MCP workflow. +- Live evidence: deterministic local server only. +- Documentation: safe reconnect matrix. +- Final disposition: pending implementation evidence. + +### A2A-001: Safe remote-task continuation + +- Current behavior: task polling is bounded, but task identity is not exposed as + a complete resumable reconciliation workflow. +- User impact: a lost response can strand externally running work. +- Security or durability impact: blind `SendMessage` resubmission duplicates a + remote task. +- Product decision: persist external task IDs before polling and resume polling + or streaming; never resubmit an ambiguous task automatically. +- Required implementation: card refresh, interface compatibility, task ID and + stream cursor persistence, auth refresh, artifact retrieval into ART-001, + cancellation, bounded retry, and EFX-001 linkage. +- Migration impact: protocol task/session records. +- Tests: ambiguous submission, polling/stream resume, card/interface change, + auth refresh, cancellation, artifacts, repair/retry/replay. +- Examples: resilient mock A2A workflow. +- Live evidence: deterministic local peer only. +- Documentation: continuation and reconciliation. +- Final disposition: pending implementation evidence. + +### PACK-001: Deterministic pack resolution and lockfile + +- Current behavior: only contained local manifests with direct integrity work. +- User impact: reusable content has no dependencies, Git/archive source, locked + graph, or offline resolution. +- Security or durability impact: ad hoc fetching weakens reproducibility. +- Product decision: support local path, pinned Git commit, and immutable HTTPS + archive sources with semantic constraints and a checked-in lockfile. No hosted + registry is required. +- Required implementation: resolver, cycles/conflicts, canonical graph, + integrity, offline/locked modes, cache, and update command. +- Migration impact: pack reference and manifest versions plus lockfile v1. +- Tests: constraints, conflicts, cycles, tamper, offline, locked drift, Git + pinning, archive limits, and path escape. +- Examples: transitive local packs and pinned archive fixture. +- Live evidence: not required. +- Documentation: source/trust/lock workflows. +- Final disposition: pending implementation evidence. + +### TRUST-001: Pack authenticity and trust policy + +- Current behavior: SHA-256 proves sameness but not publisher identity. +- User impact: users must establish provenance manually. +- Security or durability impact: a valid digest from an untrusted source can + still execute dangerous content. +- Product decision: integrate optional Sigstore-compatible bundle verification + and explicit unsigned policy. Do not invent cryptography. +- Required implementation: identity/issuer allowlists, offline bundle + verification where possible, locked digest binding, unsigned deny/warn/allow, + and process-tool trust gating. +- Migration impact: lockfile trust metadata and policy fields. +- Tests: trusted/untrusted/expired/malformed bundles, unsigned policy, digest + mismatch, and no process execution before trust. +- Examples: signed-fixture verification and explicit unsigned local pack. +- Live evidence: deterministic verification fixture. +- Documentation: trust model and keyless-signing caveats. +- Final disposition: pending implementation evidence. + +### EXT-001: Isolated extension model + +- Current behavior: no executable plugin ABI; MCP and built-in process actions + are separate surfaces. +- User impact: "plugin ABI" appears as an unresolved roadmap item. +- Security or durability impact: an in-process native ABI would undermine Rust + safety and process isolation. +- Product decision: remove native ABI from the supported surface. The supported + extension contracts are reviewed packs plus MCP or a versioned bounded process + protocol. +- Required implementation: process-protocol handshake, version negotiation, + declared schemas/capabilities, direct argv, timeout/output/cancellation, + policy, and effect identity. MCP remains the network extension option. +- Migration impact: pack action kinds and compatibility guide. +- Tests: version/schema mismatch, output overflow, timeout, cancellation, + policy, secret environment, and crash. +- Examples: local process-protocol extension. +- Live evidence: not required. +- Documentation: definitive plugin strategy and rejection of native libraries. +- Final disposition: pending redesign evidence. + +### MEM-001: Optional semantic retrieval + +- Current behavior: long-term memory supports namespace/key lookup only. +- User impact: workflows cannot retrieve relevant prior entries by text or + vectors. +- Security or durability impact: implicit model memory could bypass retention + and replay boundaries. +- Product decision: add typed entries with deterministic text search, optional + local vector/hybrid search, explicit promotion, namespaces, filters, and + retention. Retrieval remains an effect and replay uses recorded results. +- Required implementation: provider-neutral embedding interface, deterministic + fake embedder, local index, optional OpenAI adapter, external adapter trait, + filters, ranking, and explicit promotion. +- Migration impact: memory schema and index version. +- Tests: deterministic ranking, filters, namespaces, retention, repair/replay, + index rebuild, corrupt dimensions, and fake embeddings. +- Examples: hybrid retrieval and promotion. +- Live evidence: one bounded embedding scenario only if publicly exposed. +- Documentation: memory versus provider cache and working state. +- Final disposition: pending implementation evidence. + +### PROV-001: Stateless tool continuation + +- Current behavior: OpenAI/Azure tool agents reject `store: false`. +- User impact: privacy-sensitive users cannot opt out of stored provider + responses for tool loops. +- Security or durability impact: pretending support would lose reasoning and + function-call items needed for correct continuation. +- Product decision: persist provider-neutral opaque returned items needed for + stateless continuation and replay them on the next request. +- Required implementation: versioned continuation items, provider mapping, + size/redaction bounds, encryption under ENC-001, and capability negotiation. +- Migration impact: provider continuation format. +- Tests: multiple tools, reasoning items, cancellation, resume, repair session + freshness, encrypted persistence, and malformed items. +- Examples: stateless OpenAI tool workflow. +- Live evidence: one packaged `store: false` OpenAI tool run. +- Documentation: stateful versus stateless continuation. +- Final disposition: pending implementation evidence. + +## Security and operations + +### SEC-001: Stable secret-reference providers + +- Current behavior: environment references are supported; mounted file and + policy-gated command providers are absent. +- User impact: container-native secret files require wrapper scripts. +- Security or durability impact: wrappers may place resolved values in ordinary + inputs or arguments. +- Product decision: version secret references for environment, bounded mounted + file, and optional direct process provider. Resolved values never persist. +- Required implementation: policy allowlists, path containment, process argv, + timeout/output bound, redaction registration, and lifecycle zeroization where + practical. +- Migration impact: existing `{env: NAME}` remains valid. +- Tests: missing/oversized/symlink files, denied commands, timeout, redaction, + and database/trace absence. +- Examples: environment and mounted-file container secrets. +- Live evidence: OpenAI credential remains environment-only for task evidence. +- Documentation: secret reference types and threat model. +- Final disposition: pending implementation evidence. + +### NET-001: Network destination enforcement + +- Current behavior: exact/wildcard host grants and disabled redirects exist. +- User impact: users cannot constrain ports, schemes, private networks, proxies, + Unix sockets, custom CAs, or response size consistently. +- Security or durability impact: DNS rebinding, proxy routing, and local-service + access remain residual SSRF paths. +- Product decision: resolve and validate each destination against scheme, host, + port, IP class, proxy, redirect, and TLS/CA policy at the adapter boundary. +- Required implementation: resolved-IP checks, private-range controls, + rebinding defense, explicit proxy and Unix-socket denial, response limits, + shared timeouts, and protected custom-CA references. +- Migration impact: policy schema defaults preserve current public HTTPS use. +- Tests: DNS changes, private/link-local/loopback IPs, ports, schemes, + redirects, proxies, CA success/failure, oversized responses, and IPv6. +- Examples: constrained MCP/provider policies. +- Live evidence: public OpenAI route under explicit HTTPS/443 policy. +- Documentation: network model and external egress boundary. +- Final disposition: pending implementation evidence. + +### ISO-001: Honest process isolation + +- Current behavior: direct argv, cleared environment, output/time bounds, and + process-group termination exist, but policy is not an OS sandbox. +- User impact: users may overestimate allowlists. +- Security or durability impact: an allowed executable has the host identity's + full authority. +- Product decision: require an explicit isolation mode. `process` is the honest + host mode; `container` is the portable strong boundary. Optional platform + backends can be added when detected, but no weak emulation is claimed. +- Required implementation: DSL/plan visibility, fail-closed requested backend, + resource limits where supported, container invocation contract, and explicit + unsupported diagnostics on macOS/Windows/Linux backends. +- Migration impact: existing actions default to documented host-process mode. +- Tests: environment, working directory, process tree, resource bounds, + unavailable backend, and container isolation. +- Examples: host and container-isolated process action. +- Live evidence: container acceptance only. +- Documentation: policy versus isolation. +- Final disposition: pending redesign evidence. + +### BUD-001: Enforceable resource and cost budgets + +- Current behavior: per-agent turns/tokens and per-process output/time have + partial limits. +- User impact: no run-wide provider, tool, token, artifact, task, wall-time, or + cost ceiling. +- Security or durability impact: a bounded individual task can still compose + into an expensive run. +- Product decision: compile task/run budgets, reserve known units before + dispatch, reconcile actual usage after each effect, and fail safely when the + next known operation would exceed a hard bound. +- Required implementation: requests, turns, tool calls, token classes, wall + time, process output, artifact bytes, task/expansion/loop counts, and optional + versioned/custom pricing. +- Migration impact: DSL, plan, checkpoint, audit, and usage records. +- Tests: each bound, parallel reservation, unknown pricing, custom pricing, + retries, repair/replay, and off-by-one behavior. +- Examples: budget termination and usage inspection. +- Live evidence: one low request/token ceiling OpenAI scenario. +- Documentation: enforceable versus estimated limits. +- Final disposition: pending implementation evidence. + +### OCI-001: Complete container execution + +- Current behavior: OCI acceptance exists, but baseline execution on this host + failed because `/artifacts/report.txt` was rejected by workspace path policy. +- User impact: the documented separate artifact mount is not usable through the + real current acceptance path. +- Security or durability impact: weakening workspace containment would be an + unsafe fix. +- Product decision: make the artifact store part of the writable `/state` + contract and materialize exports only through an explicitly authorized + artifact-export root. Detect usable Docker/Podman-compatible engines + truthfully. +- Required implementation: runtime detection, persistent Podman handling, + non-root/read-only runs, mounted config/workspace/state/export roots, ART-001, + retry/repair/reconciliation/replay, signals, limits, CA extension, SBOM, + vulnerability/history/secret inspection, and multi-architecture builds. +- Migration impact: container mount documentation and default state paths. +- Tests: deterministic contract tests plus native/emulated execution labels. +- Examples: composite container workflow. +- Live evidence: OpenAI container scenario only when runtime remains usable. +- Documentation: runtime troubleshooting and mount migration. +- Final disposition: pending implementation evidence. + +### XPLAT-001: Hosted platform evidence + +- Current behavior: GitHub workflows are locally linted but undispatched. +- User impact: Linux x64, hosted macOS, and Windows claims lack exact-commit + evidence. +- Security or durability impact: platform-specific path, process, packaging, and + migration bugs may remain. +- Product decision: configure complete least-privilege hosted matrices but do + not claim execution during this no-push task. +- Required implementation: build/test/acceptance/package/examples/completeness + jobs for macOS ARM64, Linux ARM64/x64, Windows x64, and container/security + jobs with artifacts and digests. +- Migration impact: none. +- Tests: local workflow lint, action pin scan, and matrix completeness check. +- Examples: all public examples inventoried by jobs. +- Live evidence: not run because hosted dispatch is explicitly prohibited. +- Documentation: exact blocker and continuation. +- Final disposition: pending external-blocker evidence. + +## Removed unsupported surface + +### EVENT-001: Event triggers and calendars + +- Current behavior: external schedulers invoke the CLI. +- Product decision: remove event/calendar scheduling from the limitations list. + It is outside the deterministic single-run runtime thesis. +- Required implementation: reject any event-trigger DSL fields and keep cron, + systemd, CI, and Kubernetes invocation guides. +- Compatibility impact: no current supported syntax changes. +- Tests: strict unknown-field rejection. +- Final disposition: `removed from supported surface`. + +### DIST-001: Distributed execution and storage + +- Current behavior: one local process and SQLite are the correctness boundary. +- Product decision: distributed scheduling, leases, multi-host execution, and + distributed storage are explicit non-goals, not incomplete core behavior. +- Required implementation: remove roadmap ambiguity and make local ownership + explicit. +- Compatibility impact: none. +- Tests: documentation/product-boundary verification. +- Final disposition: `removed from supported surface`. + +### REG-001: Hosted public registry + +- Current behavior: no hosted pack service. +- Product decision: a public registry is unnecessary. PACK-001 supports local, + Git, and immutable archive sources without a hosted control plane. +- Required implementation: remove public-registry roadmap claims. +- Compatibility impact: none. +- Tests: source resolver coverage. +- Final disposition: `removed from supported surface`. + +### UI-001: Hosted UI, chat, and visual orchestration + +- Current behavior: the CLI and embeddable Rust runtime are authoritative. +- Product decision: hosted SaaS, IDE, visual editor, chat application, and + free-form conversation orchestration are explicit non-goals. +- Required implementation: reject hidden model-owned control flow and document + TEAM-001 as compiled workflow syntax. +- Compatibility impact: none. +- Tests: compiler rejects unsupported control-flow fields. +- Final disposition: `removed from supported surface`. + +## Baseline evidence + +Recorded on 2026-07-23 before framework-completeness implementation: + +- `cargo xtask verify`: passed all 12 stages. +- `cargo xtask acceptance`: passed 28 scenarios. +- `cargo xtask examples-verify`: passed. +- `cargo xtask docs-verify`: passed. +- `cargo xtask package`: passed. +- `cargo xtask secret-scan`: passed. +- `env -u OPENAI_API_KEY cargo xtask acceptance-container`: the Podman VM + required a persistent terminal to keep forwarding alive; after reaching the + OCI binary, acceptance failed with exit 3 because + `/artifacts/report.txt` escaped the authorized workspace root. No credential + was supplied and no OpenAI call occurred. From 57f9f7871195eb3e42742419d99f49ffc2b10635 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 15:25:03 +0530 Subject: [PATCH 06/44] feat: add durable content-addressed artifacts --- Cargo.lock | 33 + Cargo.toml | 1 + crates/agentctl-cli/src/main.rs | 192 ++++- crates/agentctl-core/src/policy.rs | 44 ++ crates/agentctl-runtime/src/lib.rs | 154 ++-- crates/agentctl-store/Cargo.toml | 3 +- crates/agentctl-store/src/artifact.rs | 634 ++++++++++++++++ crates/agentctl-store/src/lib.rs | 912 +++++++++++++++++++++++- docs/ARCHITECTURE.md | 4 +- docs/CONTAINER.md | 8 +- docs/DSL.md | 2 + docs/DURABLE_EXECUTION.md | 6 +- docs/LIMITATIONS.md | 3 +- docs/OPERATIONS.md | 3 +- docs/SECURITY.md | 4 +- docs/THREAT_MODEL.md | 7 +- docs/generated/CLI.md | 1 + docs/guides/repair-a-failed-workflow.md | 2 +- docs/reference/DATABASE.md | 16 +- docs/reference/ENVIRONMENT_AND_PATHS.md | 7 +- docs/research/selective-repair.md | 2 +- fuzz/Cargo.lock | 72 ++ xtask/src/acceptance.rs | 72 ++ 23 files changed, 2084 insertions(+), 98 deletions(-) create mode 100644 crates/agentctl-store/src/artifact.rs diff --git a/Cargo.lock b/Cargo.lock index 1f22d72..1fe3d25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,7 @@ version = "0.2.0" dependencies = [ "agentctl-core", "chrono", + "fs2", "hex", "parking_lot", "rusqlite", @@ -609,6 +610,16 @@ dependencies = [ "num", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.33" @@ -2444,6 +2455,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index ac52649..df2ba0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ chrono = { version = "0.4.42", default-features = false, features = ["clock", "s clap = { version = "4.5.53", features = ["derive", "env", "string"] } clap_complete = "4.5.61" futures-util = "0.3.31" +fs2 = "0.4.3" hex = "0.4.3" http = "1.4.0" jsonschema = { version = "0.37.1", default-features = false } diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index a19bb39..dfa12e8 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -103,6 +103,8 @@ enum Command { Migrate(MigrateArgs), /// Inspect and verify a local reusable pack. Packs(PackArgs), + /// Inspect, verify, export, or collect durable artifacts. + Artifacts(ArtifactArgs), /// Inspect the runtime database. Db(DbArgs), /// Read or write namespaced long-term memory. @@ -321,6 +323,44 @@ enum PackCommand { }, } +#[derive(Debug, Args)] +struct ArtifactArgs { + #[arg(long, default_value = ".agentctl/runtime.db")] + db: PathBuf, + #[command(subcommand)] + command: ArtifactCommand, +} + +#[derive(Debug, Subcommand)] +enum ArtifactCommand { + List { + #[arg(long)] + run: Option, + #[arg(long, requires = "run")] + task: Option, + }, + Inspect { + digest: String, + }, + Verify { + digest: Option, + #[arg(long, conflicts_with = "digest")] + all: bool, + }, + Export { + digest: String, + destination: PathBuf, + #[arg(long)] + overwrite: bool, + }, + Gc { + #[arg(long, default_value_t = 30)] + older_than_days: i64, + #[arg(long)] + dry_run: bool, + }, +} + #[derive(Debug, Args)] struct DbArgs { #[arg(long, default_value = ".agentctl/runtime.db")] @@ -618,12 +658,16 @@ async fn execute(cli: Cli) -> Result { let traces = store .trace_events(&args.run_id) .map_err(CliError::persistence)?; + let artifacts = store + .artifact_references(Some(&args.run_id), None) + .map_err(CliError::persistence)?; let summary = format!( - "{} {:?}; {} tasks; {} effects; {} checkpoints; {} audit events; {} traces", + "{} {:?}; {} tasks; {} effects; {} artifacts; {} checkpoints; {} audit events; {} traces", args.run_id, run.state, tasks.len(), effects.len(), + artifacts.len(), checkpoints.len(), audit.len(), traces.len(), @@ -656,6 +700,7 @@ async fn execute(cli: Cli) -> Result { "checkpoints": checkpoints, "providerSessions": provider_sessions, "toolCalls": tool_calls, + "artifacts": artifacts, "audit": audit, "traces": traces, }); @@ -669,6 +714,7 @@ async fn execute(cli: Cli) -> Result { Command::Schema(args) => schema_command(output, args), Command::Migrate(args) => migrate_command(output, args), Command::Packs(args) => pack_command(output, args), + Command::Artifacts(args) => artifact_command(output, args), Command::Db(args) => db_command(output, args), Command::Memory(args) => memory_command(output, args), Command::Gc(args) => gc_command(output, args), @@ -1288,8 +1334,12 @@ fn db_command(output: OutputFormat, args: DbArgs) -> Result { &stats, Vec::new(), format!( - "schema {}: {} runs, {} effects", - stats.schema_version, stats.runs, stats.effects + "schema {}: {} runs, {} effects, {} artifact blobs, {} artifact references", + stats.schema_version, + stats.runs, + stats.effects, + stats.artifact_blobs, + stats.artifact_references ), )?; } @@ -1306,6 +1356,141 @@ fn db_command(output: OutputFormat, args: DbArgs) -> Result { Ok(EXIT_OK) } +fn artifact_command(output: OutputFormat, args: ArtifactArgs) -> Result { + let store = open_store(&args.db)?; + match args.command { + ArtifactCommand::List { run, task } => { + let references = store + .artifact_references(run.as_deref(), task.as_deref()) + .map_err(CliError::persistence)?; + let blobs = store.artifact_blobs().map_err(CliError::persistence)?; + print_value( + output, + "ArtifactList", + &serde_json::json!({"references": references, "blobs": blobs}), + Vec::new(), + format!( + "{} artifact reference(s), {} content-addressed blob(s)", + references.len(), + blobs.len() + ), + )?; + } + ArtifactCommand::Inspect { digest } => { + let blob = store + .artifact_blob(&digest) + .map_err(CliError::persistence)?; + let references = store + .artifact_references(None, None) + .map_err(CliError::persistence)? + .into_iter() + .filter(|reference| reference.digest == digest) + .collect::>(); + print_value( + output, + "ArtifactInspection", + &serde_json::json!({"blob": blob, "references": references}), + Vec::new(), + format!( + "{}: {} bytes, {} reference(s)", + blob.digest, + blob.size_bytes, + references.len() + ), + )?; + } + ArtifactCommand::Verify { digest, all } => { + if digest.is_none() && !all { + return Err(CliError::validation( + "provide an artifact digest or use --all".to_owned(), + )); + } + let digests = if let Some(digest) = digest { + vec![digest] + } else { + store + .artifact_blobs() + .map_err(CliError::persistence)? + .into_iter() + .map(|blob| blob.digest) + .collect() + }; + let verifications = digests + .iter() + .map(|digest| store.verify_artifact(digest, Utc::now())) + .collect::, _>>() + .map_err(CliError::persistence)?; + print_value( + output, + "ArtifactVerification", + &serde_json::json!({"valid": true, "artifacts": verifications}), + Vec::new(), + format!("verified {} artifact blob(s)", verifications.len()), + )?; + } + ArtifactCommand::Export { + digest, + destination, + overwrite, + } => { + store + .export_artifact(&digest, &destination, overwrite) + .map_err(CliError::persistence)?; + print_value( + output, + "ArtifactExport", + &serde_json::json!({ + "digest": digest, + "destination": destination, + "overwritten": overwrite, + }), + Vec::new(), + format!("exported {} to {}", digest, destination.display()), + )?; + } + ArtifactCommand::Gc { + older_than_days, + dry_run, + } => { + if older_than_days < 0 { + return Err(CliError::validation( + "--older-than-days must be zero or greater".to_owned(), + )); + } + let before = Utc::now() - ChronoDuration::days(older_than_days); + let report = store + .garbage_collect_artifacts(before, dry_run) + .map_err(CliError::persistence)?; + print_value( + output, + "ArtifactGarbageCollection", + &serde_json::json!({ + "dryRun": dry_run, + "before": before, + "report": report, + }), + Vec::new(), + format!( + "{} {} artifact blob(s) and {} temporary file(s), {} reclaimable byte(s)", + if dry_run { "considered" } else { "removed" }, + if dry_run { + report.considered + } else { + u64::try_from(report.removed.len()).unwrap_or(u64::MAX) + }, + if dry_run { + report.temporary_files_considered + } else { + report.temporary_files_removed + }, + report.reclaimed_bytes + ), + )?; + } + } + Ok(EXIT_OK) +} + fn memory_command(output: OutputFormat, args: MemoryArgs) -> Result { let store = open_store(&args.db)?; match args.command { @@ -1870,6 +2055,7 @@ mod tests { "schema", "migrate", "packs", + "artifacts", "db", "memory", "gc", diff --git a/crates/agentctl-core/src/policy.rs b/crates/agentctl-core/src/policy.rs index bbd0e09..b7a3ef9 100644 --- a/crates/agentctl-core/src/policy.rs +++ b/crates/agentctl-core/src/policy.rs @@ -218,6 +218,21 @@ impl PolicyEngine { } } + pub fn resolve_artifact_path(&self, requested: &str) -> Result { + let candidate = self.join_workspace(requested)?; + let canonical = fs::canonicalize(&candidate) + .map_err(|error| PolicyError::PathEscape(format!("{requested}: {error}")))?; + if self + .writable_roots + .iter() + .any(|root| canonical.starts_with(root)) + { + Ok(canonical) + } else { + Err(PolicyError::PathEscape(requested.to_owned())) + } + } + pub fn authorize_network(&self, target: &Url) -> Result<(), PolicyError> { if target.scheme() != "https" && target.scheme() != "http" { return Err(PolicyError::NetworkDenied(target.to_string())); @@ -406,6 +421,35 @@ mod tests { ); } + #[test] + fn collects_existing_artifacts_from_an_absolute_writable_root() { + let workspace = tempdir().expect("workspace"); + let artifacts = tempdir().expect("artifacts"); + let artifact = artifacts.path().join("report.txt"); + fs::write(&artifact, b"report").expect("artifact"); + let engine = PolicyEngine::new( + PolicyDefinition { + workspace_root: workspace.path().display().to_string(), + writable_roots: vec![artifacts.path().display().to_string()], + ..PolicyDefinition::default() + }, + workspace.path(), + ) + .expect("policy"); + + assert_eq!( + engine + .resolve_artifact_path(&artifact.display().to_string()) + .expect("artifact path"), + fs::canonicalize(artifact).expect("canonical artifact") + ); + assert!( + engine + .resolve_read_path(&artifacts.path().join("report.txt").display().to_string()) + .is_err() + ); + } + #[cfg(unix)] #[test] fn rejects_symlink_escape() { diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index a0ce389..0d5abf0 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -566,6 +566,14 @@ impl Runtime { Ok((task, terminal)) }) .collect::, _>>()?; + for (task, _) in &source_tasks { + verify_artifacts(&self.store, &task.artifact_manifest).map_err(|message| { + RuntimeError::InvalidState(format!( + "recorded replay cannot verify artifacts for task `{}`: {message}", + task.task_id + )) + })?; + } let replay_id = self.ids.next_id("replay"); let trace_id = self.ids.next_id("trace"); self.store.create_run( @@ -1120,7 +1128,7 @@ impl Runtime { )); continue; } - if let Err(message) = verify_artifacts(&target_policy, &source_task.artifact_manifest) { + if let Err(message) = verify_artifacts(&self.store, &source_task.artifact_manifest) { blocked( "artifact_integrity", format!("artifact verification failed for task `{task_id}`: {message}"), @@ -1772,7 +1780,14 @@ impl Runtime { }, output_digest: versioned_json_digest(&output)?, state_delta_digest: versioned_json_digest(&delta)?, - artifact_manifest: collect_artifacts(&policy, &effects, &task.id)?, + artifact_manifest: collect_artifacts( + &self.store, + &policy, + &effects, + run_id, + &task.id, + self.clock.now(), + )?, state_delta: delta, }; self.store.complete_task( @@ -3609,9 +3624,12 @@ fn read_bounded_text_sync(path: &Path) -> Result { } fn collect_artifacts( + store: &SqliteStore, policy: &PolicyEngine, effects: &[EffectRecord], + run_id: &str, task_id: &str, + now: DateTime, ) -> Result, RuntimeError> { let mut paths = BTreeSet::new(); for effect in effects.iter().filter(|effect| { @@ -3627,19 +3645,10 @@ fn collect_artifacts( paths .into_iter() .map(|path| { - let resolved = policy.resolve_read_path(&path)?; - let metadata = std::fs::metadata(&resolved)?; - if metadata.len() > 16 * 1024 * 1024 { - return Err(RuntimeError::InvalidState(format!( - "artifact `{path}` exceeds 16777216 bytes" - ))); - } - let content = std::fs::read(&resolved)?; - Ok(ArtifactRecord { - path, - digest: format!("sha256:{}", digest(&content)), - size_bytes: metadata.len(), - }) + let resolved = policy.resolve_artifact_path(&path)?; + store + .ingest_artifact(run_id, task_id, &resolved, &path, 16 * 1024 * 1024, now) + .map_err(RuntimeError::from) }) .collect() } @@ -3663,34 +3672,15 @@ fn collect_result_paths(value: &Value, paths: &mut BTreeSet) { } } -fn verify_artifacts(policy: &PolicyEngine, artifacts: &[ArtifactRecord]) -> Result<(), String> { +fn verify_artifacts(store: &SqliteStore, artifacts: &[ArtifactRecord]) -> Result<(), String> { for artifact in artifacts { let restoration = format!( - "restore `{}` with expected digest `{}` and size {} bytes, or select its producer as an earlier repair root", - artifact.path, artifact.digest, artifact.size_bytes + "restore content-addressed blob `{}` for logical artifact `{}` with size {} bytes, import the legacy artifact, or select its producer as an earlier repair root", + artifact.digest, artifact.path, artifact.size_bytes ); - let resolved = policy - .resolve_read_path(&artifact.path) + store + .verify_artifact_record(artifact) .map_err(|error| format!("{restoration}: {error}"))?; - let metadata = - std::fs::metadata(&resolved).map_err(|error| format!("{restoration}: {error}"))?; - if metadata.len() != artifact.size_bytes { - return Err(format!( - "`{}` size mismatch: expected {} bytes, found {}; {restoration}", - artifact.path, - artifact.size_bytes, - metadata.len() - )); - } - let content = - std::fs::read(&resolved).map_err(|error| format!("{restoration}: {error}"))?; - let actual = format!("sha256:{}", digest(&content)); - if actual != artifact.digest { - return Err(format!( - "`{}` digest mismatch: expected `{}`, found `{actual}`; {restoration}", - artifact.path, artifact.digest - )); - } } Ok(()) } @@ -5325,12 +5315,10 @@ spec: } #[tokio::test] - async fn repair_blocks_when_reused_artifact_is_missing() { + async fn repair_uses_cas_after_workspace_deletion_and_blocks_blob_corruption() { let directory = tempdir().expect("tempdir"); let store = SqliteStore::open_memory().expect("store"); - let runtime = runtime(store.clone(), directory.path()).with_registry( - RuntimeRegistry::default().with_provider("fake", Arc::new(PanicProvider)), - ); + let runtime = runtime(store.clone(), directory.path()); let source_yaml = r#" apiVersion: agentctl.dev/v1alpha1 kind: Workflow @@ -5352,24 +5340,7 @@ spec: needs: [first] with: { that: false } "#; - let repaired_yaml = source_yaml - .replace( - "spec:\n", - concat!( - "spec:\n", - " providers:\n", - " fake: { kind: fake }\n", - " agents:\n", - " repaired:\n", - " provider: fake\n", - " model: fake\n", - " instructions: must never execute when the artifact is missing\n" - ), - ) - .replace( - " - id: second\n uses: action:assert\n needs: [first]\n with: { that: false }", - " - id: second\n uses: agent:repaired\n needs: [first]\n with: { prompt: repaired }", - ); + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); let (source_workflow, source_plan) = compile_fixture(source_yaml); let source_run_id = match runtime .start( @@ -5395,17 +5366,64 @@ spec: ) .expect("plan"); assert!(compatible.compatible, "{:?}", compatible.blocked_reuse); - let expected_digest = compatible.materialized_tasks[0].metadata.artifact_manifest[0] - .digest - .clone(); + let artifact = compatible.materialized_tasks[0].metadata.artifact_manifest[0].clone(); std::fs::remove_file(directory.path().join("artifact.txt")).expect("remove artifact"); + let outcome = runtime + .repair( + &repaired_workflow, + &repaired_plan, + compatible, + None, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair from CAS"); + assert_eq!(outcome.state, RunState::Succeeded); + assert!(!directory.path().join("artifact.txt").exists()); + assert!( + store + .verify_artifact_record(&artifact) + .expect("CAS blob remains valid") + .valid + ); + let references = store + .artifact_references(Some(&outcome.run_id), Some("first")) + .expect("repair artifact references"); + assert_eq!(references.len(), 1); + assert_eq!(references[0].digest, artifact.digest); + + let compatible_before_corruption = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["second".to_owned()], + false, + ) + .expect("second compatible plan"); + assert!(compatible_before_corruption.compatible); + let blob_path = store.artifact_root().join(&artifact.store_path); + let mut permissions = std::fs::metadata(&blob_path) + .expect("blob metadata") + .permissions(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + permissions.set_mode(0o600); + } + #[cfg(not(unix))] + permissions.set_readonly(false); + std::fs::set_permissions(&blob_path, permissions).expect("make blob writable"); + std::fs::write(&blob_path, b"corrupt").expect("corrupt blob"); + let stats_before = store.stats().expect("stats"); assert!(matches!( runtime .repair( &repaired_workflow, &repaired_plan, - compatible, + compatible_before_corruption, None, RunOptions::default(), &CancellationToken::new(), @@ -5414,6 +5432,8 @@ spec: Err(RuntimeError::RepairBlocked { .. }) )); assert_eq!(store.stats().expect("stats"), stats_before); + assert!(runtime.replay(&source_run_id).await.is_err()); + assert_eq!(store.stats().expect("stats"), stats_before); let plan = runtime .plan_repair( &source_run_id, @@ -5422,13 +5442,13 @@ spec: &["second".to_owned()], false, ) - .expect("blocked plan"); + .expect("blocked corrupt plan"); assert!(!plan.compatible); assert!(plan.blocked_reuse.iter().any(|block| { block.task_id == "first" && block.rule == "artifact_integrity" && block.message.contains("artifact.txt") - && block.message.contains(&expected_digest) + && block.message.contains(&artifact.digest) && block.message.contains("earlier repair root") })); assert_eq!(store.stats().expect("stats"), stats_before); diff --git a/crates/agentctl-store/Cargo.toml b/crates/agentctl-store/Cargo.toml index 41a2b08..9e096e1 100644 --- a/crates/agentctl-store/Cargo.toml +++ b/crates/agentctl-store/Cargo.toml @@ -12,16 +12,17 @@ readme.workspace = true [dependencies] agentctl-core = { version = "0.2.0", path = "../agentctl-core" } chrono.workspace = true +fs2.workspace = true hex.workspace = true parking_lot.workspace = true rusqlite.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true +tempfile.workspace = true thiserror.workspace = true [dev-dependencies] -tempfile.workspace = true [lints] workspace = true diff --git a/crates/agentctl-store/src/artifact.rs b/crates/agentctl-store/src/artifact.rs new file mode 100644 index 0000000..1a2fdc3 --- /dev/null +++ b/crates/agentctl-store/src/artifact.rs @@ -0,0 +1,634 @@ +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::SystemTime; + +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tempfile::{NamedTempFile, TempDir}; +use thiserror::Error; + +const COPY_BUFFER_BYTES: usize = 64 * 1024; + +#[derive(Debug, Error)] +pub enum ArtifactStoreError { + #[error("artifact I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("artifact `{0}` has an invalid digest")] + InvalidDigest(String), + #[error("artifact `{path}` exceeds the configured limit of {limit} bytes")] + SizeLimit { path: String, limit: u64 }, + #[error("artifact blob `{digest}` is missing at {path}")] + Missing { digest: String, path: String }, + #[error( + "artifact blob `{digest}` is corrupt: expected {expected_size} bytes and `{digest}`, found {actual_size} bytes and `{actual_digest}`" + )] + Corrupt { + digest: String, + expected_size: u64, + actual_size: u64, + actual_digest: String, + }, + #[error("artifact export target already exists: {0}")] + ExportExists(String), + #[error("artifact path is not a regular file: {0}")] + NotRegularFile(String), + #[error("artifact temporary-file persistence failed: {0}")] + Persist(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactBlob { + pub digest: String, + pub size_bytes: u64, + pub relative_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct StoredArtifactBlob { + pub digest: String, + pub size_bytes: u64, + pub modified_at: SystemTime, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactVerification { + pub digest: String, + pub size_bytes: u64, + pub path: String, + pub valid: bool, +} + +pub trait ArtifactStore: Send + Sync { + fn root(&self) -> &Path; + + fn ingest(&self, source: &Path, max_bytes: u64) -> Result; + + fn verify( + &self, + digest: &str, + expected_size: u64, + ) -> Result; + + fn export( + &self, + digest: &str, + expected_size: u64, + destination: &Path, + overwrite: bool, + ) -> Result<(), ArtifactStoreError>; +} + +#[derive(Debug, Clone)] +pub struct LocalArtifactStore { + root: PathBuf, + _temporary_root: Option>, +} + +impl LocalArtifactStore { + pub fn open(root: PathBuf) -> Result { + prepare_root(&root)?; + Ok(Self { + root, + _temporary_root: None, + }) + } + + pub(crate) fn temporary() -> Result { + let temporary_root = Arc::new(tempfile::tempdir()?); + let root = temporary_root.path().join("artifacts"); + prepare_root(&root)?; + Ok(Self { + root, + _temporary_root: Some(temporary_root), + }) + } + + fn blob_path(&self, digest: &str) -> Result { + let hex = validate_digest(digest)?; + Ok(self.root.join("sha256").join(&hex[..2]).join(hex)) + } + + pub(crate) fn lock_exclusive(&self) -> Result { + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(self.root.join(".lock"))?; + FileExt::lock_exclusive(&lock)?; + Ok(lock) + } + + pub(crate) fn stored_blobs(&self) -> Result, ArtifactStoreError> { + let mut blobs = Vec::new(); + for prefix in std::fs::read_dir(self.root.join("sha256"))? { + let prefix = prefix?; + if !prefix.file_type()?.is_dir() { + continue; + } + for entry in std::fs::read_dir(prefix.path())? { + let entry = entry?; + let metadata = entry.metadata()?; + if !metadata.is_file() { + continue; + } + let Some(hex) = entry.file_name().to_str().map(ToOwned::to_owned) else { + continue; + }; + let digest = format!("sha256:{hex}"); + if validate_digest(&digest).is_err() { + continue; + } + blobs.push(StoredArtifactBlob { + digest, + size_bytes: metadata.len(), + modified_at: metadata.modified()?, + }); + } + } + blobs.sort_by(|left, right| left.digest.cmp(&right.digest)); + Ok(blobs) + } + + pub(crate) fn stale_temporary_files( + &self, + before: SystemTime, + ) -> Result, ArtifactStoreError> { + let mut files = Vec::new(); + for entry in std::fs::read_dir(self.root.join("tmp"))? { + let entry = entry?; + let metadata = entry.metadata()?; + if metadata.is_file() && metadata.modified()? < before { + files.push((entry.path(), metadata.len())); + } + } + files.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(files) + } + + pub(crate) fn quarantined(&self) -> Result, ArtifactStoreError> { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(self.root.join("trash"))? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let name = entry.file_name(); + let Some(hex) = name.to_str() else { + continue; + }; + let digest = format!("sha256:{hex}"); + validate_digest(&digest)?; + entries.push((digest, entry.path())); + } + entries.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(entries) + } + + pub(crate) fn stage_remove(&self, digest: &str) -> Result, ArtifactStoreError> { + let source = self.blob_path(digest)?; + if !source.exists() { + return Ok(None); + } + let hex = validate_digest(digest)?; + let destination = self.root.join("trash").join(hex); + if destination.exists() { + return Err(ArtifactStoreError::Persist(format!( + "artifact quarantine target already exists: {}", + destination.display() + ))); + } + let mut permissions = std::fs::metadata(&source)?.permissions(); + make_owner_writable(&mut permissions); + std::fs::set_permissions(&source, permissions)?; + std::fs::rename(&source, &destination)?; + sync_directory(source.parent().ok_or_else(|| { + ArtifactStoreError::Persist(format!("no parent for {}", source.display())) + })?)?; + sync_directory(destination.parent().ok_or_else(|| { + ArtifactStoreError::Persist(format!("no parent for {}", destination.display())) + })?)?; + Ok(Some(destination)) + } + + pub(crate) fn restore_staged( + &self, + digest: &str, + staged: &Path, + ) -> Result<(), ArtifactStoreError> { + if !staged.exists() { + return Ok(()); + } + let destination = self.blob_path(digest)?; + let parent = destination.parent().ok_or_else(|| { + ArtifactStoreError::Persist(format!("no parent for {}", destination.display())) + })?; + std::fs::create_dir_all(parent)?; + set_private_directory(parent)?; + if destination.exists() { + std::fs::remove_file(staged)?; + } else { + std::fs::rename(staged, &destination)?; + set_blob_read_only(&destination)?; + sync_directory(parent)?; + } + Ok(()) + } + + pub(crate) fn finish_staged(&self, staged: &Path) -> Result<(), ArtifactStoreError> { + if staged.exists() { + std::fs::remove_file(staged)?; + if let Some(parent) = staged.parent() { + sync_directory(parent)?; + } + } + Ok(()) + } +} + +impl ArtifactStore for LocalArtifactStore { + fn root(&self) -> &Path { + &self.root + } + + fn ingest(&self, source: &Path, max_bytes: u64) -> Result { + let source_metadata = std::fs::symlink_metadata(source)?; + if !source_metadata.file_type().is_file() { + return Err(ArtifactStoreError::NotRegularFile( + source.display().to_string(), + )); + } + if source_metadata.len() > max_bytes { + return Err(ArtifactStoreError::SizeLimit { + path: source.display().to_string(), + limit: max_bytes, + }); + } + + let temporary_directory = self.root.join("tmp"); + let mut temporary = NamedTempFile::new_in(&temporary_directory)?; + let mut input = File::open(source)?; + let mut hasher = Sha256::new(); + let mut size_bytes = 0_u64; + let mut buffer = vec![0_u8; COPY_BUFFER_BYTES]; + loop { + let read = input.read(&mut buffer)?; + if read == 0 { + break; + } + size_bytes = size_bytes + .checked_add(u64::try_from(read).unwrap_or(u64::MAX)) + .ok_or_else(|| ArtifactStoreError::SizeLimit { + path: source.display().to_string(), + limit: max_bytes, + })?; + if size_bytes > max_bytes { + return Err(ArtifactStoreError::SizeLimit { + path: source.display().to_string(), + limit: max_bytes, + }); + } + hasher.update(&buffer[..read]); + temporary.write_all(&buffer[..read])?; + } + temporary.as_file_mut().sync_all()?; + + let digest = format!("sha256:{}", hex::encode(hasher.finalize())); + let destination = self.blob_path(&digest)?; + let parent = destination.parent().ok_or_else(|| { + ArtifactStoreError::Persist(format!("no parent for {}", destination.display())) + })?; + std::fs::create_dir_all(parent)?; + set_private_directory(parent)?; + + if destination.exists() { + self.verify(&digest, size_bytes)?; + } else { + match temporary.persist_noclobber(&destination) { + Ok(file) => { + file.sync_all()?; + set_blob_read_only(&destination)?; + sync_directory(parent)?; + } + Err(error) if destination.exists() => { + drop(error.file); + self.verify(&digest, size_bytes)?; + } + Err(error) => { + return Err(ArtifactStoreError::Persist(error.error.to_string())); + } + } + } + + let relative_path = destination + .strip_prefix(&self.root) + .map_err(|error| ArtifactStoreError::Persist(error.to_string()))? + .to_string_lossy() + .replace('\\', "/"); + Ok(ArtifactBlob { + digest, + size_bytes, + relative_path, + }) + } + + fn verify( + &self, + digest: &str, + expected_size: u64, + ) -> Result { + let path = self.blob_path(digest)?; + if !path.exists() { + return Err(ArtifactStoreError::Missing { + digest: digest.to_owned(), + path: path.display().to_string(), + }); + } + let metadata = std::fs::symlink_metadata(&path)?; + if !metadata.file_type().is_file() { + return Err(ArtifactStoreError::NotRegularFile( + path.display().to_string(), + )); + } + let mut input = File::open(&path)?; + let mut hasher = Sha256::new(); + let mut actual_size = 0_u64; + let mut buffer = vec![0_u8; COPY_BUFFER_BYTES]; + loop { + let read = input.read(&mut buffer)?; + if read == 0 { + break; + } + actual_size = actual_size.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); + hasher.update(&buffer[..read]); + } + let actual_digest = format!("sha256:{}", hex::encode(hasher.finalize())); + if actual_size != expected_size || actual_digest != digest { + return Err(ArtifactStoreError::Corrupt { + digest: digest.to_owned(), + expected_size, + actual_size, + actual_digest, + }); + } + Ok(ArtifactVerification { + digest: digest.to_owned(), + size_bytes: actual_size, + path: path.display().to_string(), + valid: true, + }) + } + + fn export( + &self, + digest: &str, + expected_size: u64, + destination: &Path, + overwrite: bool, + ) -> Result<(), ArtifactStoreError> { + let source = self.blob_path(digest)?; + self.verify(digest, expected_size)?; + if destination.exists() && !overwrite { + return Err(ArtifactStoreError::ExportExists( + destination.display().to_string(), + )); + } + if destination + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(ArtifactStoreError::NotRegularFile( + destination.display().to_string(), + )); + } + let parent = destination.parent().ok_or_else(|| { + ArtifactStoreError::Persist(format!( + "export target {} has no parent", + destination.display() + )) + })?; + std::fs::create_dir_all(parent)?; + let mut temporary = NamedTempFile::new_in(parent)?; + let mut input = File::open(source)?; + std::io::copy(&mut input, temporary.as_file_mut())?; + temporary.as_file_mut().sync_all()?; + if overwrite { + temporary + .persist(destination) + .map_err(|error| ArtifactStoreError::Persist(error.error.to_string()))?; + } else { + temporary + .persist_noclobber(destination) + .map_err(|error| ArtifactStoreError::Persist(error.error.to_string()))?; + } + sync_directory(parent)?; + Ok(()) + } +} + +fn prepare_root(root: &Path) -> Result<(), ArtifactStoreError> { + for directory in [ + root.to_path_buf(), + root.join("sha256"), + root.join("tmp"), + root.join("trash"), + ] { + std::fs::create_dir_all(&directory)?; + set_private_directory(&directory)?; + } + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(root.join(".lock"))?; + set_private_file(&lock)?; + lock.sync_all()?; + sync_directory(root)?; + Ok(()) +} + +fn validate_digest(digest: &str) -> Result<&str, ArtifactStoreError> { + let Some(hex) = digest.strip_prefix("sha256:") else { + return Err(ArtifactStoreError::InvalidDigest(digest.to_owned())); + }; + if hex.len() != 64 + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(ArtifactStoreError::InvalidDigest(digest.to_owned())); + } + Ok(hex) +} + +#[cfg(unix)] +fn set_blob_read_only(path: &Path) -> Result<(), ArtifactStoreError> { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o400))?; + Ok(()) +} + +#[cfg(not(unix))] +fn set_blob_read_only(path: &Path) -> Result<(), ArtifactStoreError> { + let mut permissions = std::fs::metadata(path)?.permissions(); + permissions.set_readonly(true); + std::fs::set_permissions(path, permissions)?; + Ok(()) +} + +#[cfg(unix)] +fn set_private_directory(path: &Path) -> Result<(), ArtifactStoreError> { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))?; + Ok(()) +} + +#[cfg(not(unix))] +fn set_private_directory(_path: &Path) -> Result<(), ArtifactStoreError> { + Ok(()) +} + +#[cfg(unix)] +fn set_private_file(file: &File) -> Result<(), ArtifactStoreError> { + use std::os::unix::fs::PermissionsExt; + + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + Ok(()) +} + +#[cfg(not(unix))] +fn set_private_file(_file: &File) -> Result<(), ArtifactStoreError> { + Ok(()) +} + +#[cfg(unix)] +fn make_owner_writable(permissions: &mut std::fs::Permissions) { + use std::os::unix::fs::PermissionsExt; + + permissions.set_mode(0o600); +} + +#[cfg(not(unix))] +fn make_owner_writable(permissions: &mut std::fs::Permissions) { + permissions.set_readonly(false); +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> Result<(), ArtifactStoreError> { + OpenOptions::new().read(true).open(path)?.sync_all()?; + Ok(()) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> Result<(), ArtifactStoreError> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ingest_deduplicates_verifies_exports_and_detects_corruption() { + let root = tempfile::tempdir().expect("root"); + let source = root.path().join("report.txt"); + std::fs::write(&source, b"durable report").expect("source"); + let store = + LocalArtifactStore::open(root.path().join("cas")).expect("artifact store opens"); + + let first = store.ingest(&source, 1024).expect("first ingest"); + let second = store.ingest(&source, 1024).expect("deduplicated ingest"); + assert_eq!(first, second); + assert!( + store + .verify(&first.digest, first.size_bytes) + .expect("verify") + .valid + ); + + let export = root.path().join("export").join("report.txt"); + store + .export(&first.digest, first.size_bytes, &export, false) + .expect("export"); + assert_eq!( + std::fs::read(&export).expect("export bytes"), + b"durable report" + ); + + let blob = store.blob_path(&first.digest).expect("blob path"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + assert_eq!( + std::fs::metadata(store.root()) + .expect("root metadata") + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + std::fs::metadata(store.root().join(".lock")) + .expect("lock metadata") + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(&blob) + .expect("blob metadata") + .permissions() + .mode() + & 0o777, + 0o400 + ); + } + let mut permissions = std::fs::metadata(&blob).expect("metadata").permissions(); + make_owner_writable(&mut permissions); + std::fs::set_permissions(&blob, permissions).expect("make writable"); + std::fs::write(&blob, b"corrupt").expect("corrupt blob"); + assert!(matches!( + store.verify(&first.digest, first.size_bytes), + Err(ArtifactStoreError::Corrupt { .. }) + )); + } + + #[test] + fn ingest_rejects_limits_and_export_symlinks() { + let root = tempfile::tempdir().expect("root"); + let source = root.path().join("large.bin"); + std::fs::write(&source, [7_u8; 8]).expect("source"); + let store = + LocalArtifactStore::open(root.path().join("cas")).expect("artifact store opens"); + assert!(matches!( + store.ingest(&source, 7), + Err(ArtifactStoreError::SizeLimit { .. }) + )); + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + + let blob = store.ingest(&source, 8).expect("ingest"); + let destination = root.path().join("destination"); + let target = root.path().join("target"); + std::fs::write(&target, b"unchanged").expect("target"); + symlink(&target, &destination).expect("symlink"); + assert!(matches!( + store.export(&blob.digest, blob.size_bytes, &destination, true), + Err(ArtifactStoreError::NotRegularFile(_)) + )); + assert_eq!(std::fs::read(&target).expect("target"), b"unchanged"); + } + } +} diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index 73de5f6..319a56c 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -1,13 +1,16 @@ //! Versioned SQLite persistence for agentctl. +pub mod artifact; + use std::collections::BTreeMap; use std::path::Path; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use agentctl_core::effect::{EffectRecord, EffectRequest, EffectStatus}; use agentctl_core::state::{RunState, TaskState}; use agentctl_core::{CompiledPlan, PLAN_FORMAT_VERSION}; +use artifact::{ArtifactStore, ArtifactStoreError, ArtifactVerification, LocalArtifactStore}; use chrono::{DateTime, Utc}; use parking_lot::Mutex; use rusqlite::{Connection, OptionalExtension, Transaction, TransactionBehavior, params}; @@ -16,10 +19,11 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use thiserror::Error; -pub const DATABASE_SCHEMA_VERSION: u32 = 5; +pub const DATABASE_SCHEMA_VERSION: u32 = 6; pub const RUNTIME_STATE_VERSION: u32 = 1; pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; pub const AUDIT_EVENT_VERSION: u32 = 1; +const ARTIFACT_INGEST_LEASE_MINUTES: i64 = 60; const MIGRATION_1: &str = r#" CREATE TABLE runs ( @@ -215,9 +219,48 @@ CREATE INDEX idx_runs_source_run ON runs(source_run_id); CREATE INDEX idx_tasks_disposition ON task_states(run_id, disposition); "#; +const MIGRATION_6: &str = r#" +CREATE TABLE artifact_blobs ( + digest TEXT PRIMARY KEY, + algorithm TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + relative_path TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL, + last_verified_at TEXT +); +CREATE TABLE artifact_refs ( + run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, + task_id TEXT NOT NULL, + logical_path TEXT NOT NULL, + logical_name TEXT NOT NULL, + media_type TEXT NOT NULL, + digest TEXT NOT NULL REFERENCES artifact_blobs(digest), + source_run_id TEXT, + source_task_id TEXT, + created_at TEXT NOT NULL, + PRIMARY KEY (run_id, task_id, logical_path), + FOREIGN KEY (run_id, task_id) REFERENCES task_states(run_id, task_id) ON DELETE CASCADE +); +CREATE INDEX idx_artifact_refs_digest ON artifact_refs(digest); +CREATE INDEX idx_artifact_refs_run_task ON artifact_refs(run_id, task_id); +CREATE TABLE artifact_ingests ( + run_id TEXT NOT NULL, + task_id TEXT NOT NULL, + logical_path TEXT NOT NULL, + digest TEXT NOT NULL REFERENCES artifact_blobs(digest) ON DELETE CASCADE, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (run_id, task_id, logical_path), + FOREIGN KEY (run_id, task_id) REFERENCES task_states(run_id, task_id) ON DELETE CASCADE +); +CREATE INDEX idx_artifact_ingests_expiry ON artifact_ingests(expires_at); +"#; + #[derive(Clone)] pub struct SqliteStore { connection: Arc>, + artifact_store: Arc, + artifact_lock: Arc>, } #[derive(Debug, Error)] @@ -244,6 +287,8 @@ pub enum StoreError { EffectNotFound(String), #[error("I/O error: {0}")] Io(#[from] std::io::Error), + #[error(transparent)] + Artifact(#[from] ArtifactStoreError), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -296,6 +341,48 @@ pub struct ArtifactRecord { pub path: String, pub digest: String, pub size_bytes: u64, + #[serde(default)] + pub media_type: String, + #[serde(default)] + pub logical_name: String, + #[serde(default)] + pub store_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactBlobRecord { + pub digest: String, + pub algorithm: String, + pub size_bytes: u64, + pub relative_path: String, + pub created_at: DateTime, + pub last_verified_at: Option>, + pub reference_count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactReference { + pub run_id: String, + pub task_id: String, + pub logical_path: String, + pub logical_name: String, + pub media_type: String, + pub digest: String, + pub source_run_id: Option, + pub source_task_id: Option, + pub created_at: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArtifactGcReport { + pub considered: u64, + pub removed: Vec, + pub reclaimed_bytes: u64, + pub temporary_files_considered: u64, + pub temporary_files_removed: u64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -447,6 +534,9 @@ pub struct DatabaseStats { pub tool_calls: i64, pub trace_events: i64, pub long_term_memory: i64, + pub artifact_blobs: i64, + pub artifact_references: i64, + pub artifact_ingests: i64, } impl SqliteStore { @@ -467,8 +557,17 @@ impl SqliteStore { } let connection = Connection::open(path)?; configure(&connection)?; + let state_root = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let artifact_store = LocalArtifactStore::open(state_root.join("artifacts"))?; + let _artifact_file_lock = artifact_store.lock_exclusive()?; + recover_artifact_quarantine(&connection, &artifact_store)?; Ok(Self { connection: Arc::new(Mutex::new(connection)), + artifact_store: Arc::new(artifact_store), + artifact_lock: Arc::new(Mutex::new(())), }) } @@ -478,6 +577,87 @@ impl SqliteStore { migrate(&mut connection)?; Ok(Self { connection: Arc::new(Mutex::new(connection)), + artifact_store: Arc::new(LocalArtifactStore::temporary()?), + artifact_lock: Arc::new(Mutex::new(())), + }) + } + + #[must_use] + pub fn artifact_root(&self) -> &Path { + self.artifact_store.root() + } + + pub fn ingest_artifact( + &self, + run_id: &str, + task_id: &str, + source: &Path, + logical_path: &str, + max_bytes: u64, + now: DateTime, + ) -> Result { + let _guard = self.artifact_lock.lock(); + let _file_lock = self.artifact_store.lock_exclusive()?; + let logical_name = Path::new(logical_path) + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .ok_or_else(|| { + StoreError::Incompatible(format!( + "artifact logical path `{logical_path}` has no valid file name" + )) + })?; + let media_type = media_type_for_path(Path::new(logical_path)); + let blob = self.artifact_store.ingest(source, max_bytes)?; + let size_bytes = i64::try_from(blob.size_bytes).map_err(|_| { + StoreError::Incompatible(format!( + "artifact `{logical_path}` size exceeds SQLite integer range" + )) + })?; + let expires_at = now + chrono::Duration::minutes(ARTIFACT_INGEST_LEASE_MINUTES); + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO artifact_blobs (digest, algorithm, size_bytes, relative_path, created_at) VALUES (?1, 'sha256', ?2, ?3, ?4) ON CONFLICT(digest) DO NOTHING", + params![ + blob.digest, + size_bytes, + blob.relative_path, + now.to_rfc3339(), + ], + )?; + let stored: (i64, String) = transaction.query_row( + "SELECT size_bytes, relative_path FROM artifact_blobs WHERE digest = ?1", + [&blob.digest], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if sqlite_u64(stored.0, "artifact_blob.size_bytes")? != blob.size_bytes + || stored.1 != blob.relative_path + { + return Err(StoreError::Corrupt(format!( + "artifact metadata for `{}` conflicts with its existing blob record", + blob.digest + ))); + } + transaction.execute( + "INSERT INTO artifact_ingests (run_id, task_id, logical_path, digest, expires_at, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT(run_id, task_id, logical_path) DO UPDATE SET digest = excluded.digest, expires_at = excluded.expires_at, created_at = excluded.created_at", + params![ + run_id, + task_id, + logical_path, + blob.digest, + expires_at.to_rfc3339(), + now.to_rfc3339(), + ], + )?; + transaction.commit()?; + Ok(ArtifactRecord { + path: logical_path.to_owned(), + digest: blob.digest, + size_bytes: blob.size_bytes, + media_type: media_type.to_owned(), + logical_name: logical_name.to_owned(), + store_path: blob.relative_path, }) } @@ -572,6 +752,14 @@ impl SqliteStore { .iter() .map(|task| (task.task_id.as_str(), task)) .collect::>(); + let _artifact_guard = self.artifact_lock.lock(); + let _artifact_file_lock = self.artifact_store.lock_exclusive()?; + for task in reused_tasks { + verify_artifact_manifest( + self.artifact_store.as_ref(), + &task.metadata.artifact_manifest, + )?; + } let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; transaction.execute( @@ -641,6 +829,15 @@ impl SqliteStore { }), now, )?; + record_artifact_references_tx( + &transaction, + run_id, + task_id, + &task.metadata.artifact_manifest, + Some(&task.source_run_id), + Some(&task.source_task_id), + now, + )?; } else { transaction.execute( "INSERT INTO task_states (run_id, task_id, position, state, disposition, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", @@ -977,6 +1174,9 @@ impl SqliteStore { now: DateTime, trace_id: &str, ) -> Result<(), StoreError> { + let _artifact_guard = self.artifact_lock.lock(); + let _artifact_file_lock = self.artifact_store.lock_exclusive()?; + verify_artifact_manifest(self.artifact_store.as_ref(), &metadata.artifact_manifest)?; let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; let current: String = transaction @@ -1013,6 +1213,19 @@ impl SqliteStore { now.to_rfc3339(), ], )?; + record_artifact_references_tx( + &transaction, + run_id, + task_id, + &metadata.artifact_manifest, + None, + None, + now, + )?; + transaction.execute( + "DELETE FROM artifact_ingests WHERE run_id = ?1 AND task_id = ?2", + params![run_id, task_id], + )?; if let Some(memory) = working_memory { transaction.execute( "UPDATE runs SET working_memory_json = ?2, updated_at = ?3 WHERE run_id = ?1", @@ -1051,6 +1264,9 @@ impl SqliteStore { now: DateTime, trace_id: &str, ) -> Result<(), StoreError> { + let _artifact_guard = self.artifact_lock.lock(); + let _artifact_file_lock = self.artifact_store.lock_exclusive()?; + verify_artifact_manifest(self.artifact_store.as_ref(), &source.artifact_manifest)?; let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; let changed = transaction.execute( @@ -1084,6 +1300,15 @@ impl SqliteStore { task_id: source.task_id.clone(), }); } + record_artifact_references_tx( + &transaction, + replay_run_id, + &source.task_id, + &source.artifact_manifest, + Some(&source.run_id), + Some(&source.task_id), + now, + )?; append_audit_tx( &transaction, replay_run_id, @@ -1882,6 +2107,303 @@ impl SqliteStore { .transpose() } + pub fn artifact_references( + &self, + run_id: Option<&str>, + task_id: Option<&str>, + ) -> Result, StoreError> { + let connection = self.connection.lock(); + let mut references = Vec::new(); + match (run_id, task_id) { + (Some(run_id), Some(task_id)) => { + let mut statement = connection.prepare( + "SELECT run_id, task_id, logical_path, logical_name, media_type, digest, source_run_id, source_task_id, created_at FROM artifact_refs WHERE run_id = ?1 AND task_id = ?2 ORDER BY logical_path", + )?; + let rows = + statement.query_map(params![run_id, task_id], decode_artifact_ref_row)?; + for row in rows { + references.push(artifact_reference_from_row(row?)?); + } + } + (Some(run_id), None) => { + let mut statement = connection.prepare( + "SELECT run_id, task_id, logical_path, logical_name, media_type, digest, source_run_id, source_task_id, created_at FROM artifact_refs WHERE run_id = ?1 ORDER BY task_id, logical_path", + )?; + let rows = statement.query_map([run_id], decode_artifact_ref_row)?; + for row in rows { + references.push(artifact_reference_from_row(row?)?); + } + } + (None, None) => { + let mut statement = connection.prepare( + "SELECT run_id, task_id, logical_path, logical_name, media_type, digest, source_run_id, source_task_id, created_at FROM artifact_refs ORDER BY run_id, task_id, logical_path", + )?; + let rows = statement.query_map([], decode_artifact_ref_row)?; + for row in rows { + references.push(artifact_reference_from_row(row?)?); + } + } + (None, Some(_)) => { + return Err(StoreError::Incompatible( + "artifact task filter requires a run filter".to_owned(), + )); + } + } + Ok(references) + } + + pub fn artifact_blobs(&self) -> Result, StoreError> { + let connection = self.connection.lock(); + let mut statement = connection.prepare( + "SELECT b.digest, b.algorithm, b.size_bytes, b.relative_path, b.created_at, b.last_verified_at, COUNT(r.digest) FROM artifact_blobs b LEFT JOIN artifact_refs r ON r.digest = b.digest GROUP BY b.digest ORDER BY b.created_at, b.digest", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, i64>(6)?, + )) + })? + .map(|row| { + let row = row?; + Ok(ArtifactBlobRecord { + digest: row.0, + algorithm: row.1, + size_bytes: sqlite_u64(row.2, "artifact_blob.size_bytes")?, + relative_path: row.3, + created_at: parse_time(&row.4, "artifact_blob.created_at")?, + last_verified_at: row + .5 + .map(|value| parse_time(&value, "artifact_blob.last_verified_at")) + .transpose()?, + reference_count: sqlite_u64(row.6, "artifact_blob.reference_count")?, + }) + }) + .collect() + } + + pub fn artifact_blob(&self, digest: &str) -> Result { + let row = self + .connection + .lock() + .query_row( + "SELECT b.digest, b.algorithm, b.size_bytes, b.relative_path, b.created_at, b.last_verified_at, (SELECT COUNT(*) FROM artifact_refs r WHERE r.digest = b.digest) FROM artifact_blobs b WHERE b.digest = ?1", + [digest], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, i64>(6)?, + )) + }, + ) + .optional()? + .ok_or_else(|| { + StoreError::Incompatible(format!("artifact blob `{digest}` was not found")) + })?; + Ok(ArtifactBlobRecord { + digest: row.0, + algorithm: row.1, + size_bytes: sqlite_u64(row.2, "artifact_blob.size_bytes")?, + relative_path: row.3, + created_at: parse_time(&row.4, "artifact_blob.created_at")?, + last_verified_at: row + .5 + .map(|value| parse_time(&value, "artifact_blob.last_verified_at")) + .transpose()?, + reference_count: sqlite_u64(row.6, "artifact_blob.reference_count")?, + }) + } + + pub fn verify_artifact( + &self, + digest: &str, + now: DateTime, + ) -> Result { + let blob = self.artifact_blob(digest)?; + let _guard = self.artifact_lock.lock(); + let _file_lock = self.artifact_store.lock_exclusive()?; + let verification = self.artifact_store.verify(digest, blob.size_bytes)?; + self.connection.lock().execute( + "UPDATE artifact_blobs SET last_verified_at = ?2 WHERE digest = ?1", + params![digest, now.to_rfc3339()], + )?; + Ok(verification) + } + + pub fn verify_artifact_record( + &self, + artifact: &ArtifactRecord, + ) -> Result { + if artifact.store_path.is_empty() { + return Err(StoreError::Incompatible(format!( + "artifact `{}` has no content-addressed blob reference", + artifact.path + ))); + } + let blob = self.artifact_blob(&artifact.digest)?; + if blob.size_bytes != artifact.size_bytes || blob.relative_path != artifact.store_path { + return Err(StoreError::Corrupt(format!( + "artifact manifest for `{}` disagrees with blob metadata `{}`", + artifact.path, artifact.digest + ))); + } + let _guard = self.artifact_lock.lock(); + let _file_lock = self.artifact_store.lock_exclusive()?; + self.artifact_store + .verify(&artifact.digest, artifact.size_bytes) + .map_err(StoreError::from) + } + + pub fn export_artifact( + &self, + digest: &str, + destination: &Path, + overwrite: bool, + ) -> Result<(), StoreError> { + let blob = self.artifact_blob(digest)?; + let _guard = self.artifact_lock.lock(); + let _file_lock = self.artifact_store.lock_exclusive()?; + self.artifact_store + .export(digest, blob.size_bytes, destination, overwrite)?; + Ok(()) + } + + pub fn garbage_collect_artifacts( + &self, + before: DateTime, + dry_run: bool, + ) -> Result { + let _artifact_guard = self.artifact_lock.lock(); + let _artifact_file_lock = self.artifact_store.lock_exclusive()?; + let stored_blobs = self.artifact_store.stored_blobs()?; + let temporary_files = self + .artifact_store + .stale_temporary_files(SystemTime::from(before))?; + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let tracked_candidates = { + let mut statement = transaction.prepare( + "SELECT b.digest, b.size_bytes FROM artifact_blobs b LEFT JOIN artifact_refs r ON r.digest = b.digest WHERE r.digest IS NULL AND b.created_at < ?1 AND NOT EXISTS (SELECT 1 FROM artifact_ingests i WHERE i.digest = b.digest AND i.expires_at > ?2) ORDER BY b.created_at, b.digest", + )?; + statement + .query_map( + params![before.to_rfc3339(), Utc::now().to_rfc3339()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + )? + .collect::, _>>()? + }; + let mut candidates = tracked_candidates + .into_iter() + .map(|(digest, size)| { + sqlite_u64(size, "artifact_gc.size_bytes").map(|size| (digest, (size, true))) + }) + .collect::, _>>()?; + let metadata_digests = { + let mut statement = transaction.prepare("SELECT digest FROM artifact_blobs")?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()? + }; + for blob in stored_blobs { + if blob.modified_at < SystemTime::from(before) + && !metadata_digests.contains(&blob.digest) + { + candidates.insert(blob.digest, (blob.size_bytes, false)); + } + } + let considered = u64::try_from(candidates.len()).unwrap_or(u64::MAX); + let reclaimed_bytes = candidates + .values() + .map(|candidate| candidate.0) + .chain(temporary_files.iter().map(|file| file.1)) + .fold(0_u64, u64::saturating_add); + let temporary_files_considered = u64::try_from(temporary_files.len()).unwrap_or(u64::MAX); + if dry_run { + transaction.rollback()?; + return Ok(ArtifactGcReport { + considered, + removed: Vec::new(), + reclaimed_bytes, + temporary_files_considered, + temporary_files_removed: 0, + }); + } + + let mut staged = Vec::new(); + for (digest, (_, tracked)) in &candidates { + match self.artifact_store.stage_remove(digest) { + Ok(Some(path)) => staged.push((digest.clone(), path)), + Ok(None) => {} + Err(error) => { + for (staged_digest, path) in &staged { + let _ = self.artifact_store.restore_staged(staged_digest, path); + } + return Err(error.into()); + } + } + if *tracked { + let changed = match transaction.execute( + "DELETE FROM artifact_blobs WHERE digest = ?1 AND NOT EXISTS (SELECT 1 FROM artifact_refs WHERE digest = ?1) AND NOT EXISTS (SELECT 1 FROM artifact_ingests WHERE digest = ?1 AND expires_at > ?2)", + params![digest, Utc::now().to_rfc3339()], + ) { + Ok(changed) => changed, + Err(error) => { + for (staged_digest, path) in &staged { + let _ = self.artifact_store.restore_staged(staged_digest, path); + } + return Err(error.into()); + } + }; + if changed != 1 { + for (staged_digest, path) in &staged { + let _ = self.artifact_store.restore_staged(staged_digest, path); + } + return Err(StoreError::Incompatible(format!( + "artifact `{digest}` became reachable during garbage collection" + ))); + } + } + } + if let Err(error) = transaction.execute( + "DELETE FROM artifact_ingests WHERE expires_at <= ?1", + [Utc::now().to_rfc3339()], + ) { + for (digest, path) in &staged { + let _ = self.artifact_store.restore_staged(digest, path); + } + return Err(error.into()); + } + if let Err(error) = transaction.commit() { + for (digest, path) in &staged { + let _ = self.artifact_store.restore_staged(digest, path); + } + return Err(error.into()); + } + for (_, path) in &staged { + self.artifact_store.finish_staged(path)?; + } + for (path, _) in &temporary_files { + std::fs::remove_file(path)?; + } + Ok(ArtifactGcReport { + considered, + removed: candidates.keys().cloned().collect(), + reclaimed_bytes, + temporary_files_considered, + temporary_files_removed: temporary_files_considered, + }) + } + pub fn garbage_collect(&self, before: DateTime) -> Result { let connection = self.connection.lock(); let expired = connection.execute( @@ -1920,6 +2442,9 @@ impl SqliteStore { "tool_calls", "trace_events", "long_term_memory", + "artifact_blobs", + "artifact_refs", + "artifact_ingests", ]; if !allowed.contains(&table) { return Err(StoreError::Incompatible( @@ -1946,6 +2471,9 @@ impl SqliteStore { tool_calls: count("tool_calls")?, trace_events: count("trace_events")?, long_term_memory: count("long_term_memory")?, + artifact_blobs: count("artifact_blobs")?, + artifact_references: count("artifact_refs")?, + artifact_ingests: count("artifact_ingests")?, }) } @@ -1976,6 +2504,7 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { (3_u32, MIGRATION_3), (4_u32, MIGRATION_4), (5_u32, MIGRATION_5), + (6_u32, MIGRATION_6), ]; for (version, sql) in migrations .into_iter() @@ -2056,6 +2585,166 @@ fn append_audit_tx( Ok(()) } +type ArtifactReferenceRow = ( + String, + String, + String, + String, + String, + String, + Option, + Option, + String, +); + +fn decode_artifact_ref_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + )) +} + +fn artifact_reference_from_row(row: ArtifactReferenceRow) -> Result { + Ok(ArtifactReference { + run_id: row.0, + task_id: row.1, + logical_path: row.2, + logical_name: row.3, + media_type: row.4, + digest: row.5, + source_run_id: row.6, + source_task_id: row.7, + created_at: parse_time(&row.8, "artifact_reference.created_at")?, + }) +} + +fn verify_artifact_manifest( + artifact_store: &dyn ArtifactStore, + artifacts: &[ArtifactRecord], +) -> Result<(), StoreError> { + for artifact in artifacts { + if artifact.store_path.is_empty() { + return Err(StoreError::Incompatible(format!( + "artifact `{}` has not been imported into the content-addressed store", + artifact.path + ))); + } + artifact_store.verify(&artifact.digest, artifact.size_bytes)?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn record_artifact_references_tx( + transaction: &Transaction<'_>, + run_id: &str, + task_id: &str, + artifacts: &[ArtifactRecord], + source_run_id: Option<&str>, + source_task_id: Option<&str>, + now: DateTime, +) -> Result<(), StoreError> { + for artifact in artifacts { + if artifact.store_path.is_empty() { + continue; + } + let size_bytes = i64::try_from(artifact.size_bytes).map_err(|_| { + StoreError::Incompatible(format!( + "artifact `{}` size exceeds SQLite integer range", + artifact.path + )) + })?; + transaction.execute( + "INSERT INTO artifact_blobs (digest, algorithm, size_bytes, relative_path, created_at) VALUES (?1, 'sha256', ?2, ?3, ?4) ON CONFLICT(digest) DO NOTHING", + params![ + artifact.digest, + size_bytes, + artifact.store_path, + now.to_rfc3339(), + ], + )?; + let stored: (i64, String) = transaction.query_row( + "SELECT size_bytes, relative_path FROM artifact_blobs WHERE digest = ?1", + [&artifact.digest], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if sqlite_u64(stored.0, "artifact_blob.size_bytes")? != artifact.size_bytes + || stored.1 != artifact.store_path + { + return Err(StoreError::Corrupt(format!( + "artifact metadata for `{}` conflicts with its existing blob record", + artifact.digest + ))); + } + transaction.execute( + "INSERT INTO artifact_refs (run_id, task_id, logical_path, logical_name, media_type, digest, source_run_id, source_task_id, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + run_id, + task_id, + artifact.path, + artifact.logical_name, + artifact.media_type, + artifact.digest, + source_run_id, + source_task_id, + now.to_rfc3339(), + ], + )?; + } + Ok(()) +} + +fn recover_artifact_quarantine( + connection: &Connection, + artifact_store: &LocalArtifactStore, +) -> Result<(), StoreError> { + for (digest, path) in artifact_store.quarantined()? { + let retained: bool = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM artifact_blobs WHERE digest = ?1)", + [&digest], + |row| row.get(0), + )?; + if retained { + artifact_store.restore_staged(&digest, &path)?; + } else { + artifact_store.finish_staged(&path)?; + } + } + Ok(()) +} + +fn media_type_for_path(path: &Path) -> &'static str { + match path + .extension() + .and_then(|extension| extension.to_str()) + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("json") => "application/json", + Some("yaml" | "yml") => "application/yaml", + Some("md" | "txt" | "log") => "text/plain", + Some("html" | "htm") => "text/html", + Some("csv") => "text/csv", + Some("pdf") => "application/pdf", + Some("png") => "image/png", + Some("jpg" | "jpeg") => "image/jpeg", + Some("svg") => "image/svg+xml", + _ => "application/octet-stream", + } +} + +fn sqlite_u64(value: i64, field: &str) -> Result { + u64::try_from(value) + .map_err(|_| StoreError::Corrupt(format!("{field} cannot be negative: {value}"))) +} + fn encode(value: &T) -> Result { serde_json::to_string(value).map_err(StoreError::from) } @@ -2127,6 +2816,58 @@ spec: .expect("create run"); } + fn begin_task(store: &SqliteStore, run_id: &str) { + store + .transition_task( + run_id, + "one", + TaskState::Ready, + None, + None, + None, + Utc::now(), + "trace", + ) + .expect("ready"); + store + .transition_task( + run_id, + "one", + TaskState::Running, + None, + None, + None, + Utc::now(), + "trace", + ) + .expect("running"); + } + + fn complete_with_artifact(store: &SqliteStore, run_id: &str, artifact: ArtifactRecord) { + store + .complete_task( + run_id, + "one", + &serde_json::json!({"ok": true}), + None, + &TaskCompletionMetadata { + execution: TaskExecutionMetadata { + metadata_version: 1, + definition_fingerprint: "definition".to_owned(), + input_digest: "input".to_owned(), + output_contract_fingerprint: "contract".to_owned(), + }, + output_digest: "output".to_owned(), + state_delta: serde_json::json!({}), + state_delta_digest: "state".to_owned(), + artifact_manifest: vec![artifact], + }, + Utc::now(), + "trace", + ) + .expect("complete task"); + } + fn create_version_four_database(path: &Path) { let connection = Connection::open(path).expect("raw connection"); connection.execute_batch(MIGRATION_1).expect("v1 schema"); @@ -2524,4 +3265,171 @@ spec: Some(serde_json::json!(2)) ); } + + #[test] + fn artifacts_are_deduplicated_referenced_and_durable_without_workspace_files() { + let directory = tempdir().expect("temp dir"); + let database = directory.path().join("state").join("runtime.db"); + let source = directory.path().join("workspace").join("report.txt"); + std::fs::create_dir_all(source.parent().expect("source parent")).expect("workspace"); + std::fs::write(&source, b"durable report").expect("source"); + let store = SqliteStore::open(&database).expect("store"); + + create(&store, "first"); + begin_task(&store, "first"); + let first = store + .ingest_artifact("first", "one", &source, "report.txt", 1024, Utc::now()) + .expect("first ingest"); + assert_eq!(store.stats().expect("stats").artifact_ingests, 1); + complete_with_artifact(&store, "first", first.clone()); + assert_eq!(store.stats().expect("stats").artifact_ingests, 0); + + create(&store, "second"); + begin_task(&store, "second"); + let second = store + .ingest_artifact("second", "one", &source, "report.json", 1024, Utc::now()) + .expect("deduplicated ingest"); + assert_eq!(first.digest, second.digest); + complete_with_artifact(&store, "second", second); + assert_eq!(store.stats().expect("stats").artifact_blobs, 1); + let references = store.artifact_references(None, None).expect("references"); + assert_eq!(references.len(), 2); + assert_eq!(references[0].media_type, "text/plain"); + assert_eq!(references[1].media_type, "application/json"); + + std::fs::remove_file(&source).expect("remove workspace source"); + drop(store); + let reopened = SqliteStore::open(&database).expect("reopen"); + let verification = reopened + .verify_artifact(&first.digest, Utc::now()) + .expect("verify without workspace"); + assert!(verification.valid); + let export = directory.path().join("export").join("report.txt"); + reopened + .export_artifact(&first.digest, &export, false) + .expect("export"); + assert_eq!( + std::fs::read(export).expect("export bytes"), + b"durable report" + ); + } + + #[test] + fn concurrent_ingests_share_one_blob_and_keep_independent_leases() { + let directory = tempdir().expect("temp dir"); + let database = directory.path().join("state").join("runtime.db"); + let source = directory.path().join("source.bin"); + std::fs::write(&source, b"same bytes").expect("source"); + let store = SqliteStore::open(&database).expect("store"); + create(&store, "left"); + create(&store, "right"); + drop(store); + + let workers = ["left", "right"].map(|run_id| { + let database = database.clone(); + let source = source.clone(); + std::thread::spawn(move || { + let store = SqliteStore::open(&database).expect("worker store"); + store + .ingest_artifact( + run_id, + "one", + &source, + &format!("{run_id}.bin"), + 1024, + Utc::now(), + ) + .expect("concurrent ingest") + }) + }); + let [left_worker, right_worker] = workers; + let left = left_worker.join().expect("left worker"); + let right = right_worker.join().expect("right worker"); + assert_eq!(left.digest, right.digest); + let store = SqliteStore::open(&database).expect("reopen"); + let stats = store.stats().expect("stats"); + assert_eq!(stats.artifact_blobs, 1); + assert_eq!(stats.artifact_ingests, 2); + } + + #[test] + fn artifact_gc_respects_leases_references_orphans_and_partial_files() { + let directory = tempdir().expect("temp dir"); + let database = directory.path().join("state").join("runtime.db"); + let source = directory.path().join("artifact.bin"); + std::fs::write(&source, b"leased").expect("source"); + let store = SqliteStore::open(&database).expect("store"); + create(&store, "leased"); + let leased = store + .ingest_artifact("leased", "one", &source, "leased.bin", 1024, Utc::now()) + .expect("leased ingest"); + let future = Utc::now() + chrono::Duration::days(1); + let protected = store + .garbage_collect_artifacts(future, false) + .expect("lease-protected gc"); + assert!(protected.removed.is_empty()); + store + .verify_artifact(&leased.digest, Utc::now()) + .expect("leased blob remains"); + + store + .connection() + .execute( + "UPDATE artifact_ingests SET expires_at = ?1", + [(Utc::now() - chrono::Duration::seconds(1)).to_rfc3339()], + ) + .expect("expire lease"); + let removed = store + .garbage_collect_artifacts(future, false) + .expect("expired lease gc"); + assert_eq!(removed.removed, [leased.digest]); + + let orphan = store + .artifact_store + .ingest(&source, 1024) + .expect("orphan blob"); + let partial = store.artifact_root().join("tmp").join("interrupted"); + std::fs::write(&partial, b"partial").expect("partial file"); + let preview = store + .garbage_collect_artifacts(future, true) + .expect("dry run"); + assert_eq!(preview.considered, 1); + assert_eq!(preview.temporary_files_considered, 1); + assert_eq!(preview.temporary_files_removed, 0); + assert!(partial.exists()); + let collected = store + .garbage_collect_artifacts(future, false) + .expect("orphan gc"); + assert_eq!(collected.removed, [orphan.digest]); + assert_eq!(collected.temporary_files_removed, 1); + assert!(!partial.exists()); + } + + #[test] + fn artifact_quarantine_is_recovered_after_interrupted_gc() { + let directory = tempdir().expect("temp dir"); + let database = directory.path().join("state").join("runtime.db"); + let source = directory.path().join("artifact.bin"); + std::fs::write(&source, b"recoverable").expect("source"); + let store = SqliteStore::open(&database).expect("store"); + create(&store, "run"); + begin_task(&store, "run"); + let artifact = store + .ingest_artifact("run", "one", &source, "artifact.bin", 1024, Utc::now()) + .expect("ingest"); + complete_with_artifact(&store, "run", artifact.clone()); + let staged = store + .artifact_store + .stage_remove(&artifact.digest) + .expect("stage") + .expect("staged path"); + assert!(staged.exists()); + drop(store); + + let reopened = SqliteStore::open(&database).expect("recover quarantine"); + reopened + .verify_artifact(&artifact.digest, Utc::now()) + .expect("restored referenced artifact"); + assert!(!staged.exists()); + } } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 59c3c25..6cf8a90 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -19,7 +19,7 @@ Parsing produces a versioned `Workflow`; compilation resolves references, valida An effect request is persisted before any filesystem observation/mutation, process, internal-memory update, long-term-memory operation, tool, model, MCP, or A2A call. Its stable identity covers run, task, task attempt, ordinal, operation, and input digest. A confirmed result can be reused on resume. A started but unconfirmed effect is uncertain and stops recovery rather than being repeated. -State transitions, checkpoint creation, working-memory replacement, and audit insertion are transactionally coupled where consistency requires it. Provider continuation, function-call correlation, effects, approvals, and redacted trace events are inspectable through the public CLI. Long-term memory is a separate table and never participates in replay correctness. OpenTelemetry export remains optional and is not the audit log. +State transitions, checkpoint creation, working-memory replacement, artifact references, and audit insertion are transactionally coupled where consistency requires it. Artifact bytes live in an immutable SHA-256 content-addressed store beside SQLite; durable leases and a cross-process lock coordinate ingestion with reachability GC. Provider continuation, function-call correlation, effects, approvals, artifacts, and redacted trace events are inspectable through the public CLI. Long-term memory is a separate table and never participates in replay correctness. OpenTelemetry export remains optional and is not the audit log. ## Determinism and concurrency @@ -31,6 +31,6 @@ Clock and identifier generation are injected. Provider responses, tools, and ext The workspace uses Rust edition 2024, pins Rust 1.88 as the MSRV, forbids unsafe code, and denies clippy warnings. HTTP uses rustls and disables redirects. Subprocesses use direct argv, a cleared environment, explicit allowlists, validated timeout/output limits, concurrent bounded pipe draining, cancellation, and kill/reap cleanup. SQLite is bundled for predictable installation and creates private files on Unix. SIGINT and SIGTERM converge on durable cancellation. -The OCI build is multi-stage: only the optimized Rust binary enters a maintained distroless runtime with CA roots and a non-root identity. `/config` is workflow configuration, `/workspace` is the read-only working tree, `/state` holds SQLite, and `/artifacts` receives declared outputs. State must be mounted again for inspect/resume/replay. The root filesystem may be read-only. See [Container contract](CONTAINER.md) and ADR 0007. +The OCI build is multi-stage: only the optimized Rust binary enters a maintained distroless runtime with CA roots and a non-root identity. `/config` is workflow configuration, `/workspace` is the read-only working tree, `/state` holds SQLite and the content-addressed artifact store, and `/artifacts` receives declared workflow outputs. State must be mounted again for inspect/resume/replay/repair and artifact export. The root filesystem may be read-only. See [Container contract](CONTAINER.md) and ADR 0007. See the [architecture diagrams](architecture/DIAGRAMS.md), [ADRs](adr/), and [Durable execution](DURABLE_EXECUTION.md) for failure semantics. diff --git a/docs/CONTAINER.md b/docs/CONTAINER.md index 14b6a70..18163b4 100644 --- a/docs/CONTAINER.md +++ b/docs/CONTAINER.md @@ -21,10 +21,10 @@ The `Containerfile` combines the secret with public roots on a tmpfs mount for t | --- | --- | | `/config` | read-only reviewed workflow and pack configuration | | `/workspace` | usually read-only source/fixture workspace | -| `/state` | writable SQLite database and durable recovery state | -| `/artifacts` | writable declared workflow artifacts | +| `/state` | writable SQLite database, CAS blobs, and durable recovery state | +| `/artifacts` | writable declared workflow output/export surface | -Pass workflow values with repeated `--input KEY=VALUE`, `--inputs-file`, or `--inputs` JSON. Prefer files for large or sensitive non-provider inputs. Provider credentials are environment references only; never put a key in CLI arguments, YAML, an image layer, or an ordinary input value. Before a bind-mount run, provision `/state` and `/artifacts` host directories so UID/GID 65532 can write them and the runner's artifact collector can read them. Durable state may contain prompts and outputs; protect it like a sensitive build artifact. +Pass workflow values with repeated `--input KEY=VALUE`, `--inputs-file`, or `--inputs` JSON. Prefer files for large or sensitive non-provider inputs. Provider credentials are environment references only; never put a key in CLI arguments, YAML, an image layer, or an ordinary input value. Before a bind-mount run, provision `/state` and `/artifacts` host directories so UID/GID 65532 can write them. Successful bounded workflow files are copied into `/state/artifacts/sha256`; `/artifacts` remains the convenient CI collection surface. Durable state may contain prompts, outputs, and artifact bytes; protect it like a sensitive build artifact. The image emits exactly one versioned JSON result on stdout with `--output json`; failures emit one versioned JSON error on stderr. The document includes exit status semantics, run/trace IDs, final state, and declared outputs. Progress is not mixed into stdout. Persist `/state` for later `inspect`, approval resolution, `resume`, `replay`, or `repair`. @@ -46,7 +46,7 @@ docker run --rm --read-only --user 65532:65532 \ The value form `--env OPENAI_API_KEY` forwards an already protected host variable without placing its value in the command. The credential-free container acceptance uses the same command with the fake provider and without that environment variable. -For selective repair, mount the corrected workflow under `/config`, keep the source database under `/state`, and retain any workspace artifacts required by upstream reuse. Plan without forwarding provider credentials: +For selective repair, mount the corrected workflow under `/config` and keep the source database plus its `/state/artifacts` CAS under `/state`. The original workspace output can be absent after successful ingestion. Plan without forwarding provider credentials: ```console docker run --rm --read-only --user 65532:65532 --network none \ diff --git a/docs/DSL.md b/docs/DSL.md index 7f9d90c..081f6b2 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -10,6 +10,8 @@ Task output is JSON. Built-in actions own an object contract, agents can declare Providers, action environments, and protocol headers use `{ env: NAME }` secret references. Secret names are validated and values never become the workflow document. +`policy.workspaceRoot` is the default boundary for relative file paths. Each `writableRoots` entry may be workspace-relative or an explicit absolute mount such as `/artifacts`. Ordinary reads remain workspace-confined. After a successful authorized mutation, the runtime may read that exact output through its writable-root boundary to ingest the bounded regular file into durable CAS; this does not grant tasks general read access to the external root. + The compiler validates missing references, duplicate tasks, cycles, task-aware templates, tool references, provider capabilities, agent limits, and sequential runtime settings before execution. Ready tasks follow declaration order. `maxConcurrency` must be `1` in this version. `builtin.shell.exec` captures stdout and stderr concurrently. Its optional `stdoutLimitBytes`, `stderrLimitBytes`, and `combinedOutputLimitBytes` fields default to 1 MiB, 1 MiB, and 2 MiB respectively. Each configured value must be between 1 byte and 16 MiB. `timeoutSeconds` must be between 1 and 86,400. Exceeding an output bound terminates and reaps the process and records a structured failed effect; timeout or cancellation remains an uncertain effect because external changes may already have occurred. These fields are validated identically for workflow and pack actions. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index fff2dab..35cf5cd 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -16,12 +16,14 @@ Pure operations need no external guarantee. Idempotent and keyed effects may be Working-memory replacement, the task transition, checkpoint, and audit event commit in one SQLite transaction. Tool-effect and tool-call terminal status also commit together, so inspection cannot observe one as completed while the other remains started. On resume, a confirmed memory-write effect is applied to the reconstructed working-memory value during the succeeding transition. Long-term memory is an external effect and is not rolled back by replay. -Successful task completion also commits repair metadata atomically: definition fingerprint, resolved-input digest, output-contract fingerprint, output digest, immutable state delta and digest, artifact manifest, audit event, and checkpoint. Repair initialization starts from target initial memory and applies only reused successful task deltas in topological order. It never copies a terminal source's final memory snapshot. +Successful workspace mutations are ingested into the local content-addressed artifact store before task completion. Ingestion uses an atomic temporary file, SHA-256 identity, immutable deduplicated blobs, a cross-process lock, and a durable one-hour lease. Successful task completion then commits the artifact references with the definition fingerprint, resolved-input digest, output-contract fingerprint, output digest, immutable state delta and digest, audit event, and checkpoint, and releases the ingestion lease in the same SQLite transaction. Repair initialization starts from target initial memory and applies only reused successful task deltas in topological order. It never copies a terminal source's final memory snapshot. -Repair planning is effect-free. A source task is reusable only when its metadata version, definition, dependencies, resolved inputs, output contract/value, state delta, artifacts, and effect certainty are compatible. The repair run stores the reused result and provenance in its own task row, so later source-row garbage collection does not break it. Artifact bytes remain a separately retained workspace responsibility. +Repair planning is effect-free. A source task is reusable only when its metadata version, definition, dependencies, resolved inputs, output contract/value, state delta, content-addressed artifacts, and effect certainty are compatible. The repair run stores the reused result, artifact references, and provenance in its own rows, so later source-row or workspace deletion does not break it. Missing or corrupt CAS bytes block reuse before a repair run is created. Cancellation is both an injected token and a durable run flag. CLI SIGINT and SIGTERM cancel in-flight async calls and return exit `130`; `agentctl cancel` records a request for another process to observe. An overall CLI deadline can be set with `--timeout-seconds`, in addition to task/tool/provider/protocol bounds. A provider, tool, process, MCP, or A2A timeout/cancellation/transport loss after dispatch marks the effect `uncertain`; resume refuses to guess and requires reconciliation or an explicit fork. A repaired agent task starts a fresh provider session. Source `previous_response_id`, incomplete turns, pending tool calls, and reasoning state are not copied. Validated task output and reconstructed memory are the only cross-task/cross-run dataflow. Clock and ID generation are injected; test providers/tools/protocol handlers are injected. The current scheduler is sequential, so output and memory commit order is task declaration order. + +The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index d28be49..ec6aef3 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -14,6 +14,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an - Timeout/transport ambiguity is not automatically retried; confirmed effects survive resume; call IDs are scoped by run; missing credentials fail before run/database creation. - Non-interactive approvals durably pause, signals cancel safely, JSON errors include available run/trace correlation, and SQLite uses WAL plus a bounded lock wait. - The packaged CLI, clean-directory quickstart, cron-like empty environment, and non-root/read-only OCI contract have executable acceptance coverage. +- Successful bounded file outputs are atomically ingested into a local immutable content-addressed store with durable references, verification/export commands, lease-safe reachability GC, interrupted-GC recovery, and local/OCI acceptance coverage. - Shell execution and acceptance/container helpers use bounded concurrent capture. Output overflow terminates/reaps the child with a structured secret-safe error; timeouts and cancellation retain durable uncertain-effect semantics. - Hosted workflows use least privilege, full-SHA action pins with version comments, complete-history/tree Gitleaks, deterministic fake-secret detection, dependency/image scans, and required production/image CycloneDX artifacts with digests. @@ -45,7 +46,7 @@ These are useful extensions but are not required by the product thesis. They nee - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. - At-most-once model/remote calls can become uncertain in the dispatch/acknowledgement window. Inspect and reconcile externally; use `fork` only when fresh effects are knowingly acceptable. - Selective repair requires task metadata version 1. Successful tasks from databases created before schema 5 remain inspectable but must execute from an earlier repair root or a full fork. -- Automatic artifact manifests cover bounded files reported by successful workspace-mutation results. Artifact bytes are not copied into SQLite or a content-addressed store; retain the configured workspace. Missing, moved, size-mismatched, or digest-mismatched bytes block repair before run creation and report the expected artifact identity. +- Automatic artifact ingestion covers regular files up to 16 MiB reported by successful built-in workspace-mutation results. Larger outputs and artifacts produced only by opaque external effects require an explicit bounded import/export integration. The local CAS must be backed up with SQLite; missing or corrupt blob bytes block repair before run creation and report the expected artifact identity. - A confirmed non-idempotent mutation in a repair closure remains blocked. The only built-in reconciliation outcome is an operator-confirmed `not-applied` result for a started or uncertain effect; compensation and provider-specific deduplication workflows are not implemented. - Retry remains a bounded same-run task policy. There is no separate command that creates a new terminal-source retry run for an unchanged workflow; use repair with an unchanged target definition and explicit roots when its compatibility checks fit. - Tool-using OpenAI/Azure agents require stored-response continuation. `store: false` is rejected until stateless response-item replay is implemented. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 6194336..5f9b9c7 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -63,7 +63,8 @@ A oneshot service has one active invocation at a time. Use distinct databases on 5. Use `repair TARGET SOURCE --from TASK --plan` before executing a corrected terminal workflow from a task boundary. 6. Use `fork` for a broader new run that may execute fresh effects. 7. For an uncertain effect, reconcile the remote system first. The runtime intentionally refuses unsafe resume or repair. +8. Verify retained bytes with `agentctl artifacts --db PATH verify --all`; export a digest with `agentctl artifacts --db PATH export DIGEST DESTINATION`. Repair planning exits `3` when compatibility or effect safety blocks reuse. Read `blockedReuse`, choose an earlier/additional root, restore a verified artifact, or reconcile an effect. Do not bypass the plan with a fresh fork unless repeating all effects is an intentional operator decision. -Use `agentctl gc --db PATH --older-than-days N` for expired memory and old terminal histories after the organization's retention/backup requirements are satisfied. SQLite WAL files belong with the database during backup. A future schedule-run key may improve deduplication; today the external scheduler owns overlap prevention. +Use `agentctl gc --db PATH --older-than-days N` for expired memory and old terminal histories. Then use `agentctl artifacts --db PATH gc --older-than-days N --dry-run` to preview unreferenced blobs before running it without `--dry-run`. SQLite WAL files and the sibling artifact root belong together during backup. A future schedule-run key may improve deduplication; today the external scheduler owns overlap prevention. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index d870168..b830b19 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -10,7 +10,7 @@ - Tool input and output JSON Schemas are enforced. Models, MCP annotations, A2A cards, remote schemas, and results cannot grant capabilities. - Requests are ledgered before effects. Global denial or approval cannot be weakened by a tool contract. Approval is durable; non-interactive mode pauses with exit `3` or uses an explicitly stricter deny/fail mode, never a prompt or implicit approval. - SQLite uses foreign keys, WAL/busy timeout, version checks, checksummed checkpoints, and mode `0600` on Unix. -- Repair never mutates a terminal source. Reuse requires versioned definition/input/contract/output/state metadata and verified artifact paths, sizes, and SHA-256 digests. Repair creation and reused-task materialization are one SQLite transaction. +- Repair never mutates a terminal source. Reuse requires versioned definition/input/contract/output/state metadata and verified content-addressed artifact sizes and SHA-256 digests. Artifact ingestion uses atomic no-clobber writes, immutable blobs, bounded leases, and a cross-process GC lock. Repair creation and reused-task/reference materialization are one SQLite transaction. - A recorded replay cannot be a repair source because it has no direct effect ledger. A materialized reused/recorded task cannot be selected for restart without returning to direct effect history. Repaired agents start fresh provider sessions. - Packs require a supported manifest/version and can be checked against SHA-256 integrity. - The workspace forbids unsafe Rust, denies warnings, locks dependencies, checks licenses/sources/advisories, scans secret patterns, and keeps live tests outside CI. @@ -23,6 +23,6 @@ Prompts, file content, model output, remote artifacts, and tool output may be co MCP reconnection and A2A resubmission are intentionally not automatic. Streaming is bounded but completed results, not token deltas, enter workflow state. Windows cannot express Unix database mode bits; rely on the user profile ACL and CI tests. -SQLite file access is the repair authorization boundary. There is no tenant identity or row-level authorization. Artifact bytes remain in the workspace rather than SQLite; losing or changing them blocks reuse but does not restore them automatically. +SQLite and sibling artifact-root access are the repair authorization boundary. There is no tenant identity or row-level authorization. Artifact bytes are not encrypted; an identity that can modify the state directory can corrupt or replace local history, although digest verification prevents silent reuse of changed bytes. Report vulnerabilities privately to the repository maintainer. Do not include credentials, database contents, or production prompts in a report. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 87deee6..2aa43a9 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -2,7 +2,7 @@ ## Assets and boundaries -Assets are workspace files, allowed environment secrets, provider accounts, external systems reached by tools, workflow history, prompts/results, approvals, and the integrity of deterministic scheduling. Boundaries are the YAML/pack parser, filesystem/process/network executors, provider APIs, MCP servers, A2A peers, SQLite, trace exporters, and dependencies. +Assets are workspace files, content-addressed artifact bytes, allowed environment secrets, provider accounts, external systems reached by tools, workflow history, prompts/results, approvals, and the integrity of deterministic scheduling. Boundaries are the YAML/pack parser, filesystem/process/network executors, provider APIs, MCP servers, A2A peers, SQLite and its sibling artifact root, trace exporters, and dependencies. The local operator and reviewed binary are trusted. Workflow authors are only as trusted as policy grants. Models, file content, remote descriptions/results, pack content without independent provenance, and all network peers are untrusted. The host OS, CA store, and Rust dependency supply chain are assumed but monitored dependencies. @@ -17,10 +17,11 @@ The local operator and reviewed binary are trusted. Workflow authors are only as | MCP annotation or A2A card claims safety | always treated as untrusted metadata | compromised authorized peer can return malicious but schema-valid data | | Crash duplicates an external mutation | request-before-start ledger, uncertain state, no silent retry | external action may have happened without acknowledgement | | Replay reissues effects | recorded replay uses stored terminal output only | replayed data may no longer reflect current reality, by design | -| Repair reuses tampered or unrelated state | stable workflow identity, versioned task/input/contract/output/state fingerprints, artifact digest checks, transactional materialization | an attacker with database/workspace write access is inside the local application trust boundary | +| Repair reuses tampered or unrelated state | stable workflow identity, versioned task/input/contract/output/state fingerprints, immutable CAS blobs, artifact digest checks, transactional materialization | an attacker with database/artifact-root write access is inside the local application trust boundary | | Repair duplicates a partial mutation | closure effect inspection, conservative uncertainty block, narrow operator `not-applied` reconciliation | remote truth may remain unknowable and keep the repair blocked | | Repair carries failed model state | every repaired agent starts a fresh provider session; dataflow uses validated JSON output | a valid reused output can still contain hostile content and must remain policy constrained | -| Source deletion breaks repair | reused output/state/artifact metadata is materialized into the repair run | artifact bytes still require durable workspace retention | +| Source or workspace deletion breaks repair | reused output/state metadata and independent CAS references are materialized into the repair run | deleting/corrupting the shared CAS or restoring SQLite without it still blocks repair | +| Concurrent GC removes an in-flight artifact | cross-process lock, durable ingestion leases, transactional references, quarantine recovery | network filesystems with broken advisory-lock semantics are unsupported | | Approval bypass in CI | non-interactive durable pause or explicit deny/fail; operator resolution | stolen database write access is outside application trust boundary | | Pack substitution | SHA-256 verification and semver/API checks | digest source/signature trust is manual | | Corrupt or future state misexecutes | schema/version/checksum/deserialization failures | SQLite file deletion or rollback by an attacker is not prevented | diff --git a/docs/generated/CLI.md b/docs/generated/CLI.md index 307063f..3c17cfd 100644 --- a/docs/generated/CLI.md +++ b/docs/generated/CLI.md @@ -26,6 +26,7 @@ Commands: schema Print or write the generated workflow JSON Schema migrate Translate an unversioned TypeScript-era workflow into v1alpha1 packs Inspect and verify a local reusable pack + artifacts Inspect, verify, export, or collect durable artifacts db Inspect the runtime database memory Read or write namespaced long-term memory gc Garbage-collect expired memory and old terminal runs diff --git a/docs/guides/repair-a-failed-workflow.md b/docs/guides/repair-a-failed-workflow.md index 7749c5a..c0bfc1d 100644 --- a/docs/guides/repair-a-failed-workflow.md +++ b/docs/guides/repair-a-failed-workflow.md @@ -182,7 +182,7 @@ Recorded replay has a new replay run ID but the same semantic outputs. It dispat | `output_contract_mismatch` | The target expects a different contract. | Rerun from the producer. | | `output_digest_mismatch` | Persisted output was modified or corrupted. | Do not reuse it; rerun from the producer. | | `state_delta_missing` or `state_delta_invalid` | Successful boundary-state metadata is absent or corrupt. | Select the task as an earlier root; do not edit the database. | -| `artifact_integrity` | An artifact is missing, changed, or outside policy. The block reports its path, expected digest, and expected size. | Restore the exact retained artifact or select its producer as an earlier repair root. | +| `artifact_integrity` | A content-addressed artifact is missing or corrupt. The block reports its logical path, expected digest, and expected size. | Restore the database and sibling CAS from a consistent backup, or select its producer as an earlier repair root. | | `unresolved_reused_effect` | A nominally successful reusable task retains a started or uncertain effect. | Reconcile external reality before reuse. | | `legacy_task_metadata` | The source predates repair metadata v1. | Use an earlier root or a full fork. | | `new_task_outside_repair_closure` | A new unrelated task has no result. | Add it as a root or choose an earlier common boundary. | diff --git a/docs/reference/DATABASE.md b/docs/reference/DATABASE.md index ce2d142..0ffb5b1 100644 --- a/docs/reference/DATABASE.md +++ b/docs/reference/DATABASE.md @@ -1,6 +1,6 @@ # Runtime database and migrations -The local SQLite database is both history and part of the correctness boundary. The current database schema version is `5`. +The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `6`. ## Stored records @@ -12,12 +12,13 @@ The local SQLite database is both history and part of the correctness boundary. - ordered audit and trace events - provider sessions and tool calls - namespaced long-term memory with optional expiry +- content-addressed blob metadata, logical run/task references, provenance, verification time, and bounded ingestion leases Working memory is stored on the run and in checkpoints. Provider credentials are not stored. Other confidential content may be stored, including prompts, tool output, and remote artifacts. -Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. A repair transaction creates the run, materializes every reused task, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete a repair run. +Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. A repair transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete a repair run. -Artifact manifests contain policy-resolved paths, byte sizes, and SHA-256 digests. The bytes remain in the configured durable workspace. Retain that workspace for as long as a task result may be repaired or audited. +Artifact manifests contain logical path/name, media type, byte size, SHA-256 digest, and CAS-relative path. Blob bytes live under `/artifacts/sha256/`; identical content is stored once. A completed repair/replay receives its own references, so source-row and workspace deletion do not break it. ## Migrations @@ -26,20 +27,25 @@ The store reads SQLite `user_version` and applies forward migrations in order in ```text agentctl db stats --db .agentctl/runtime.db --output json --color never agentctl db migrate --db .agentctl/runtime.db --output json --color never +agentctl artifacts --db .agentctl/runtime.db list --run RUN_ID --output json +agentctl artifacts --db .agentctl/runtime.db verify --all --output json +agentctl artifacts --db .agentctl/runtime.db export SHA256_DIGEST ./report.bin +agentctl artifacts --db .agentctl/runtime.db gc --older-than-days 30 --dry-run ``` `db migrate` may write the database. Back up the database and its WAL state before an upgrade. ## Locking and permissions -The connection enables foreign keys, WAL mode, and a five-second busy timeout. Unix database files use mode `0600`. Windows relies on user-profile ACLs. Separate runs can share a database, but this is not a distributed lease and does not prevent two runs from changing the same external resource. +The connection enables foreign keys, WAL mode, and a five-second busy timeout. Unix database files use mode `0600`. Windows relies on user-profile ACLs. Artifact ingestion and GC use an advisory cross-process lock plus SQLite leases; this protects the local store but is not a distributed lease. Separate runs can share a database but can still change the same external resource. ## Backups and recovery -Use an SQLite-aware online backup or stop writers before copying the database and WAL files. Restore the set consistently. Do not use ordinary file synchronization that can separate a database from uncheckpointed WAL content. +Use an SQLite-aware online backup or stop writers before copying the database, WAL files, and sibling `artifacts/` directory. Restore the set consistently. Do not use ordinary file synchronization that can separate SQLite from uncheckpointed WAL content or the artifact bytes referenced by it. Delete old terminal history only after retention requirements are met: ```text agentctl gc --db .agentctl/runtime.db --older-than-days 30 --output json --color never +agentctl artifacts --db .agentctl/runtime.db gc --older-than-days 30 --output json --color never ``` diff --git a/docs/reference/ENVIRONMENT_AND_PATHS.md b/docs/reference/ENVIRONMENT_AND_PATHS.md index 5af4c36..95a4dcc 100644 --- a/docs/reference/ENVIRONMENT_AND_PATHS.md +++ b/docs/reference/ENVIRONMENT_AND_PATHS.md @@ -29,7 +29,8 @@ Normal `cargo xtask docs-verify`, `cargo xtask verify`, and `cargo xtask accepta | Workflow file | positional argument | Read-only input, at most 1 MiB. | | Workspace | current directory | Override with `--workspace`. | | Runtime database | `.agentctl/runtime.db` | Override with `--db`; SQLite WAL belongs to the same state set. | -| Artifact path | workflow-defined | Must remain under a policy-approved writable root. | +| CAS artifact root | `/artifacts` | Immutable SHA-256 blobs; back up with SQLite. | +| Workflow output path | workflow-defined | Must remain under a policy-approved writable root; successful bounded files are ingested into CAS. | ## Container paths @@ -37,8 +38,8 @@ Normal `cargo xtask docs-verify`, `cargo xtask verify`, and `cargo xtask accepta | --- | --- | | `/config` | reviewed read-only configuration | | `/workspace` | normally read-only workspace | -| `/state` | writable durable state | -| `/artifacts` | writable collected output | +| `/state` | writable SQLite and content-addressed durable state | +| `/artifacts` | writable workflow output/export mount | | `/tmp` | small runtime tmpfs when the root filesystem is read-only | State and artifacts must be writable by UID/GID 65532 in the production image. diff --git a/docs/research/selective-repair.md b/docs/research/selective-repair.md index c9d22c4..45f1da3 100644 --- a/docs/research/selective-repair.md +++ b/docs/research/selective-repair.md @@ -16,7 +16,7 @@ Selective repair is a new execution from a task boundary, not history replay. Th | Create repair state atomically | [SQLite transactions](https://www.sqlite.org/lang_transaction.html) | Adopt | An immediate transaction either creates the run, all task materializations, audit events, and checkpoint, or none of them. | Schema migration 5 adds repair lineage and task reuse metadata; repair creation uses one transaction. | Planning/materialization failure cannot leave a runnable partial repair. | Databases migrate forward; unknown newer versions still fail explicitly. | | Start a repaired agent cleanly | [OpenAI conversation state](https://developers.openai.com/api/docs/guides/conversation-state) | Adopt | `previous_response_id` continues one task-local Responses conversation; it is not a cross-run dataflow mechanism. | A repaired task starts a new provider session. Continuation is used only between turns of that new task. | Failed response IDs, pending tool calls, and uncommitted reasoning are never copied. | Provider continuation within ordinary tasks remains unchanged. | | Preserve tool-call correlation | [OpenAI Responses migration guidance](https://developers.openai.com/api/docs/guides/migrate-to-responses#additional-differences) | Adopt | Function-call output must correlate to the returned call ID, and stateful continuation has explicit rules. | Repair uses the existing strict tool schema, call-ID, response-ID, and bounded-turn implementation. | Reused upstream data flows through validated JSON output, not hidden conversation state. | OpenAI tool-using agents still require stored continuation in v1alpha1. | -| Keep artifacts trustworthy | [Bazel remote cache protocol](https://bazel.build/remote/caching#remote-caching) | Adapt | Content-addressed output metadata is useful even without adopting a remote cache. | Successful workspace mutations record bounded path, size, and SHA-256 metadata; planning re-resolves the path and verifies size/digest. | Missing, changed, oversized, or path-escaping artifacts block reuse. | Artifact bytes remain in the configured durable workspace; no remote CAS is added. | +| Keep artifacts trustworthy | [Bazel remote cache protocol](https://bazel.build/remote/caching#remote-caching) | Adapt | Content-addressed output identity and immutable bytes make reuse independent of a mutable workspace. | Successful workspace mutations atomically ingest bounded bytes into a local SHA-256 CAS; SQLite retains logical metadata, provenance, references, and leases. | Missing, corrupt, oversized, or unreferenced artifacts block reuse or are collected safely. | A remote CAS is not added; SQLite and the local artifact root are backed up together. | ## Patterns rejected diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 5745a36..4f10c60 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -89,12 +89,14 @@ version = "0.2.0" dependencies = [ "agentctl-core", "chrono", + "fs2", "hex", "parking_lot", "rusqlite", "serde", "serde_json", "sha2", + "tempfile", "thiserror", ] @@ -389,6 +391,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -431,6 +439,16 @@ dependencies = [ "num", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures-channel" version = "0.3.33" @@ -902,6 +920,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -1384,6 +1408,19 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.42" @@ -1678,6 +1715,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.19" @@ -2053,6 +2103,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 8e7a36c..9997150 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -169,6 +169,7 @@ pub fn run(root: &Path) -> Result<()> { ensure!(array_len(&mock_inspect, "/data/checkpoints")? > 0); ensure!(array_len(&mock_inspect, "/data/audit")? > 0); ensure!(array_len(&mock_inspect, "/data/traces")? > 0); + ensure!(array_len(&mock_inspect, "/data/artifacts")? == 1); ensure!( mock_inspect .pointer("/data/toolCalls/0/callId") @@ -177,6 +178,76 @@ pub fn run(root: &Path) -> Result<()> { .pointer("/data/toolCalls/0/effectId") .and_then(Value::as_str) ); + let artifact_digest = string_at(&mock_inspect, "/data/artifacts/0/digest")?; + let artifact_list = successful_json( + &binary, + &mock_workspace, + &strings([ + "artifacts", + "--db", + path(&mock_db)?, + "list", + "--run", + mock_id, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure!(array_len(&artifact_list, "/data/references")? == 1); + ensure!(array_len(&artifact_list, "/data/blobs")? == 1); + let artifact_inspect = successful_json( + &binary, + &mock_workspace, + &strings([ + "artifacts", + "--db", + path(&mock_db)?, + "inspect", + artifact_digest, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&artifact_inspect, "/data/blob/referenceCount", 1_u64)?; + fs::remove_file(mock_workspace.join("artifacts/mock-report.txt"))?; + let artifact_verify = successful_json( + &binary, + &mock_workspace, + &strings([ + "artifacts", + "--db", + path(&mock_db)?, + "verify", + artifact_digest, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&artifact_verify, "/data/valid", true)?; + let exported_artifact = mock_workspace.join("exports/mock-report.txt"); + successful_json( + &binary, + &mock_workspace, + &strings([ + "artifacts", + "--db", + path(&mock_db)?, + "export", + artifact_digest, + path(&exported_artifact)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure!(fs::read_to_string(exported_artifact)? == VERIFY_TOKEN); scenario( 7, @@ -425,6 +496,7 @@ pub fn run(root: &Path) -> Result<()> { ensure!(replay_id != mock_id); let replay_inspect = inspect(&binary, &mock_workspace, &mock_db, replay_id)?; ensure!(array_len(&replay_inspect, "/data/effects")? == 0); + ensure!(array_len(&replay_inspect, "/data/artifacts")? == 1); let failed_replay = json_with_code( &binary, &workspace, From a1bf666ef9c9aab2b33e480ff4d704ebf331d804 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 15:57:25 +0530 Subject: [PATCH 07/44] feat: add run upgrade and effect reconciliation --- Cargo.lock | 1 + README.md | 1 + crates/agentctl-cli/src/main.rs | 221 ++- crates/agentctl-runtime/src/lib.rs | 1629 ++++++++++++++++++- crates/agentctl-store/src/lib.rs | 1001 +++++++++++- docs/DURABLE_EXECUTION.md | 4 +- docs/LIMITATIONS.md | 4 +- docs/execution/COMPLETENESS_VERIFICATION.md | 6 +- docs/execution/LIMITATION_BURNDOWN.md | 41 +- docs/generated/CLI.md | 202 ++- docs/guides/EFFECT_RECONCILIATION.md | 60 + docs/guides/LEGACY_RUN_UPGRADE.md | 39 + docs/guides/repair-a-failed-workflow.md | 11 +- docs/reference/DATABASE.md | 10 +- xtask/Cargo.toml | 1 + xtask/src/acceptance.rs | 72 +- xtask/src/main.rs | 29 +- 17 files changed, 3191 insertions(+), 141 deletions(-) create mode 100644 docs/guides/EFFECT_RECONCILIATION.md create mode 100644 docs/guides/LEGACY_RUN_UPGRADE.md diff --git a/Cargo.lock b/Cargo.lock index 1fe3d25..a1e2490 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2660,6 +2660,7 @@ dependencies = [ "anyhow", "hex", "nix", + "rusqlite", "serde_json", "sha2", "tempfile", diff --git a/README.md b/README.md index 024d585..f962f54 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task ``` See [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. +For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). ## Safety boundary diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index dfa12e8..9484713 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -17,8 +17,12 @@ use agentctl_protocols::{A2aClient, McpClient, ProtocolActionHandler, ProtocolHt use agentctl_providers::{ AnthropicProvider, FakeProvider, GoogleProvider, HttpProviderConfig, OpenAiProvider, }; -use agentctl_runtime::{BuiltinToolExecutor, RunOptions, Runtime, RuntimeRegistry}; -use agentctl_store::{ApprovalResolution, RunMode, SqliteStore, StoreError, TaskDisposition}; +use agentctl_runtime::{ + BuiltinToolExecutor, EffectReconciliationInput, RunOptions, Runtime, RuntimeRegistry, +}; +use agentctl_store::{ + ApprovalResolution, ReconciliationStatus, RunMode, SqliteStore, StoreError, TaskDisposition, +}; use chrono::{Duration as ChronoDuration, Utc}; use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum}; use clap_complete::Shell; @@ -85,6 +89,8 @@ enum Command { Fork(ForkArgs), /// Create a new run that reuses compatible upstream results and executes a repaired suffix. Repair(RepairArgs), + /// Analyze or upgrade retained legacy run records for selective reuse. + Runs(RunsArgs), /// Durably request cancellation. Cancel(RunIdArgs), /// Inspect durable run, task, and audit state. @@ -208,6 +214,26 @@ struct RepairArgs { timeout_seconds: Option, } +#[derive(Debug, Args)] +struct RunsArgs { + #[arg(long, default_value = ".agentctl/runtime.db")] + db: PathBuf, + #[command(subcommand)] + command: RunsCommand, +} + +#[derive(Debug, Subcommand)] +enum RunsCommand { + /// Prove reusable legacy metadata without changing the source run. + Analyze { run_id: String }, + /// Transactionally persist every legacy field that can be proven. + Upgrade { + run_id: String, + #[arg(long)] + dry_run: bool, + }, +} + #[derive(Debug, Args)] struct ApprovalArgs { #[arg(long, default_value = ".agentctl/runtime.db")] @@ -226,25 +252,40 @@ struct EffectArgs { #[derive(Debug, Subcommand)] enum EffectCommand { - Inspect { + List { run_id: String, #[arg(long)] task: Option, }, + Inspect { + effect_id: String, + }, Reconcile { effect_id: String, #[arg(long, value_enum)] - outcome: ReconciledOutcome, + status: ReconciledOutcome, #[arg(long, default_value = "cli-user")] actor: String, #[arg(long)] reason: String, + #[arg(long)] + evidence_file: Option, + #[arg(long)] + result_file: Option, + #[arg(long)] + result_schema_file: Option, + #[arg(long)] + compensation_effect: Option, + #[arg(long)] + approved: bool, }, } #[derive(Debug, Clone, Copy, ValueEnum)] enum ReconciledOutcome { + Applied, NotApplied, + Compensated, } #[derive(Debug, Subcommand)] @@ -615,6 +656,7 @@ async fn execute(cli: Cli) -> Result { print_outcome(output, &outcome) } Command::Repair(args) => repair_workflow(output, args).await, + Command::Runs(args) => runs_command(output, args), Command::Cancel(args) => { let store = open_store(&args.db)?; store @@ -643,6 +685,9 @@ async fn execute(cli: Cli) -> Result { let effects = store .list_effects(&args.run_id) .map_err(CliError::persistence)?; + let effect_reconciliations = store + .run_effect_reconciliations(&args.run_id) + .map_err(CliError::persistence)?; let approvals = store .pending_approvals(&args.run_id) .map_err(CliError::persistence)?; @@ -696,6 +741,7 @@ async fn execute(cli: Cli) -> Result { "run": run, "tasks": tasks, "effects": effects, + "effectReconciliations": effect_reconciliations, "approvals": approvals, "checkpoints": checkpoints, "providerSessions": provider_sessions, @@ -1013,7 +1059,7 @@ fn approval_command(output: OutputFormat, args: ApprovalArgs) -> Result Result { let store = open_store(&args.db)?; match args.command { - EffectCommand::Inspect { run_id, task } => { + EffectCommand::List { run_id, task } => { let effects = store .list_effects(&run_id) .map_err(CliError::persistence)? @@ -1023,38 +1069,111 @@ fn effect_command(output: OutputFormat, args: EffectArgs) -> Result>(); + let reconciliations = store + .run_effect_reconciliations(&run_id) + .map_err(CliError::persistence)?; print_value( output, - "EffectInspection", + "EffectList", &serde_json::json!({ "runId": run_id, "taskId": task, "effects": effects, + "reconciliations": reconciliations, + }), + Vec::new(), + format!( + "{} effect(s), {} reconciliation record(s)", + effects.len(), + reconciliations.len() + ), + )?; + } + EffectCommand::Inspect { effect_id } => { + let effect = store + .load_effect(&effect_id) + .map_err(CliError::persistence)?; + let reconciliations = store + .effect_reconciliations(&effect_id) + .map_err(CliError::persistence)?; + print_value( + output, + "EffectInspection", + &serde_json::json!({ + "effect": effect, + "reconciliations": reconciliations, + "effectiveReconciliation": reconciliations.last(), }), Vec::new(), - format!("{} effect(s)", effects.len()), + format!( + "{} {:?}; {} reconciliation record(s)", + effect.request.id, + effect.status, + reconciliations.len() + ), )?; } EffectCommand::Reconcile { effect_id, - outcome: ReconciledOutcome::NotApplied, + status, actor, reason, + evidence_file, + result_file, + result_schema_file, + compensation_effect, + approved, } => { - store - .reconcile_effect_not_applied(&effect_id, &actor, &reason, Utc::now()) + let effect = store + .load_effect(&effect_id) + .map_err(CliError::persistence)?; + let run = store + .load_run(&effect.request.run_id) .map_err(CliError::persistence)?; + let workflow: Workflow = serde_json::from_value(run.workflow) + .map_err(|error| CliError::persistence(error.to_string()))?; + let base = resolve_base_path(run.base_path.as_deref().map(Path::new))?; + let registry = build_registry(&workflow, &base)?; + let evidence = evidence_file + .as_deref() + .map(read_json) + .transpose()? + .unwrap_or_else(|| { + serde_json::json!({ + "kind": "operator_statement", + "statement": reason, + }) + }); + let result = result_file.as_deref().map(read_json).transpose()?; + let result_schema = result_schema_file.as_deref().map(read_json).transpose()?; + let status = match status { + ReconciledOutcome::Applied => ReconciliationStatus::Applied, + ReconciledOutcome::NotApplied => ReconciliationStatus::NotApplied, + ReconciledOutcome::Compensated => ReconciliationStatus::Compensated, + }; + let runtime = Runtime::new(store, base).with_registry(registry); + let reconciliation = runtime + .reconcile_effect(EffectReconciliationInput { + effect_id, + status, + actor, + reason, + evidence, + result, + result_schema, + compensation_effect_id: compensation_effect, + approved, + }) + .map_err(map_runtime_error)?; print_value( output, "EffectReconciliation", - &serde_json::json!({ - "effectId": effect_id, - "outcome": "not_applied", - "actor": actor, - "reason": reason, - }), + &reconciliation, Vec::new(), - "effect reconciled as not applied; a compatible repair may now retry it".to_owned(), + format!( + "{} reconciled as {:?} by {}", + reconciliation.effect_id, reconciliation.status, reconciliation.actor + ), )?; } } @@ -1289,6 +1408,64 @@ fn pack_command(output: OutputFormat, args: PackArgs) -> Result { Ok(EXIT_OK) } +fn runs_command(output: OutputFormat, args: RunsArgs) -> Result { + let store = open_store(&args.db)?; + let runtime = Runtime::new(store, current_dir()?); + match args.command { + RunsCommand::Analyze { run_id } + | RunsCommand::Upgrade { + run_id, + dry_run: true, + } => { + let analysis = runtime + .analyze_legacy_run(&run_id) + .map_err(map_runtime_error)?; + let roots = if analysis.recommended_repair_roots.is_empty() { + "none".to_owned() + } else { + analysis.recommended_repair_roots.join(",") + }; + print_value( + output, + "LegacyRunUpgradeAnalysis", + &analysis, + Vec::new(), + format!( + "{}: {} upgradeable, {} unavailable; safe repair roots: {roots}", + analysis.run_id, + analysis.upgradeable_tasks.len(), + analysis.unavailable_tasks.len(), + ), + )?; + } + RunsCommand::Upgrade { + run_id, + dry_run: false, + } => { + let result = runtime + .upgrade_legacy_run(&run_id) + .map_err(map_runtime_error)?; + let roots = if result.analysis_after.recommended_repair_roots.is_empty() { + "none".to_owned() + } else { + result.analysis_after.recommended_repair_roots.join(",") + }; + print_value( + output, + "LegacyRunUpgrade", + &result, + Vec::new(), + format!( + "{}: upgraded {} task(s); safe repair roots: {roots}", + result.run_id, + result.upgraded_tasks.len(), + ), + )?; + } + } + Ok(EXIT_OK) +} + fn cancellation_token(timeout_seconds: Option) -> CancellationToken { let token = CancellationToken::new(); let signal = token.clone(); @@ -1334,10 +1511,12 @@ fn db_command(output: OutputFormat, args: DbArgs) -> Result { &stats, Vec::new(), format!( - "schema {}: {} runs, {} effects, {} artifact blobs, {} artifact references", + "schema {}: {} runs, {} effects, {} reconciliations, {} run upgrades, {} artifact blobs, {} artifact references", stats.schema_version, stats.runs, stats.effects, + stats.effect_reconciliations, + stats.run_upgrades, stats.artifact_blobs, stats.artifact_references ), @@ -1876,6 +2055,11 @@ fn read_text(path: &Path) -> Result { Ok(content) } +fn read_json(path: &Path) -> Result { + serde_json::from_str(&read_text(path)?) + .map_err(|error| CliError::validation(format!("{}: {error}", path.display()))) +} + fn write_text(path: &Path, content: &str) -> Result<(), CliError> { if let Some(parent) = path.parent() && !parent.as_os_str().is_empty() @@ -2046,6 +2230,7 @@ mod tests { "replay", "fork", "repair", + "runs", "cancel", "inspect", "effects", diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index 0d5abf0..fd1e22a 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -23,8 +23,10 @@ use agentctl_core::template::{EvalContext, TemplateError, evaluate_when, render} use agentctl_core::tool::{ToolContract, ToolContractError, ToolExecutor}; use agentctl_observability::{NoopTraceSink, SpanKind, TraceEvent, TracePhase, TraceSink}; use agentctl_store::{ - ApprovalRequest, ArtifactRecord, ReusedTaskMaterialization, RunMode, SqliteStore, StoreError, - TaskCompletionMetadata, TaskDisposition, TaskExecutionMetadata, TaskRecord, + ApprovalRequest, ArtifactRecord, CheckpointRecord, EffectReconciliationRecord, + EffectReconciliationRequest, LegacyTaskUpgrade, ReconciliationStatus, + ReusedTaskMaterialization, RunMode, SqliteStore, StoreError, TaskCompletionMetadata, + TaskDisposition, TaskExecutionMetadata, TaskRecord, }; use async_trait::async_trait; use chrono::{DateTime, Utc}; @@ -77,7 +79,17 @@ pub trait ExternalActionHandler: Send + Sync { ) -> Result; } +pub trait EffectReconciliationHook: Send + Sync { + fn validate( + &self, + effect: &EffectRecord, + evidence: &Value, + result: Option<&Value>, + ) -> Result<(), String>; +} + const MAX_WORKSPACE_FILE_BYTES: u64 = 1024 * 1024; +const MAX_ARTIFACT_BYTES: u64 = 16 * 1024 * 1024; pub struct BuiltinToolExecutor { contract: ToolContract, @@ -182,6 +194,7 @@ impl ToolExecutor for BuiltinToolExecutor { pub struct RuntimeRegistry { providers: BTreeMap>, tools: BTreeMap>, + reconciliation_hooks: BTreeMap>, external_actions: Option>, } @@ -202,6 +215,16 @@ impl RuntimeRegistry { self } + #[must_use] + pub fn with_reconciliation_hook( + mut self, + operation: impl Into, + hook: Arc, + ) -> Self { + self.reconciliation_hooks.insert(operation.into(), hook); + self + } + #[must_use] pub fn with_external_actions(mut self, handler: Arc) -> Self { self.external_actions = Some(handler); @@ -309,6 +332,60 @@ pub struct RepairOutcome { pub output: Option, } +pub const LEGACY_UPGRADE_ANALYSIS_VERSION: &str = "agentctl.dev/legacy-upgrade/v1"; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTaskUpgradeAnalysis { + pub task_id: String, + pub state: TaskState, + pub already_current: bool, + pub upgradeable: bool, + pub confidence: String, + pub reasons: Vec, + pub provenance: BTreeMap, + pub proposed_metadata: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyRunUpgradeAnalysis { + pub api_version: String, + pub run_id: String, + pub database_schema_version: u32, + pub terminal: bool, + pub fully_upgradeable: bool, + pub already_current: bool, + pub upgradeable_tasks: Vec, + pub unavailable_tasks: Vec, + pub recommended_repair_roots: Vec, + pub tasks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyRunUpgradeResult { + pub upgrade_id: String, + pub run_id: String, + pub upgraded_tasks: Vec, + pub analysis_before: LegacyRunUpgradeAnalysis, + pub analysis_after: LegacyRunUpgradeAnalysis, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EffectReconciliationInput { + pub effect_id: String, + pub status: ReconciliationStatus, + pub actor: String, + pub reason: String, + pub evidence: Value, + pub result: Option, + pub result_schema: Option, + pub compensation_effect_id: Option, + pub approved: bool, +} + #[derive(Debug, Error)] pub enum RuntimeError { #[error(transparent)] @@ -459,6 +536,7 @@ impl Runtime { run.state ))); } + self.prepare_reconciled_resume(run_id, &trace_id)?; let tasks = self.store.list_tasks(run_id)?; for task in tasks .iter() @@ -537,6 +615,69 @@ impl Runtime { self.drive(run_id, &trace_id, options, cancellation).await } + fn prepare_reconciled_resume(&self, run_id: &str, trace_id: &str) -> Result<(), RuntimeError> { + for effect in self.store.list_effects(run_id)? { + let Some(reconciliation) = self + .store + .latest_effect_reconciliation(&effect.request.id)? + else { + continue; + }; + if !matches!( + reconciliation.status, + ReconciliationStatus::NotApplied | ReconciliationStatus::Compensated + ) { + continue; + } + let Some(task) = self + .store + .list_tasks(run_id)? + .into_iter() + .find(|task| task.task_id == effect.request.task_id) + else { + return Err(RuntimeError::InvalidState(format!( + "reconciled effect `{}` has no task row", + effect.request.id + ))); + }; + if task.state == TaskState::WaitingForEffect { + self.store.transition_task( + run_id, + &task.task_id, + TaskState::Running, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + } + if matches!(task.state, TaskState::Running | TaskState::WaitingForEffect) { + self.store.transition_task( + run_id, + &task.task_id, + TaskState::RetryScheduled, + None, + Some("operator reconciliation permits a fresh task attempt"), + None, + self.clock.now(), + trace_id, + )?; + self.store.transition_task( + run_id, + &task.task_id, + TaskState::Ready, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + } + } + Ok(()) + } + pub async fn replay(&self, source_run_id: &str) -> Result { let source = self.store.load_run(source_run_id)?; let source_tasks = self.store.list_tasks(source_run_id)?; @@ -650,6 +791,373 @@ impl Runtime { }) } + pub fn analyze_legacy_run( + &self, + run_id: &str, + ) -> Result { + self.analyze_legacy_run_internal(run_id) + .map(|(analysis, _)| analysis) + } + + pub fn upgrade_legacy_run(&self, run_id: &str) -> Result { + let (analysis_before, mut updates) = self.analyze_legacy_run_internal(run_id)?; + let source = self.store.load_run(run_id)?; + let workflow: Workflow = serde_json::from_value(source.workflow)?; + let base_path = source + .base_path + .as_deref() + .map(PathBuf::from) + .unwrap_or_else(|| self.base_path.clone()); + let policy = PolicyEngine::new(workflow.spec.policy, &base_path)?; + let now = self.clock.now(); + for update in &mut updates { + for artifact in &mut update.metadata.artifact_manifest { + if artifact.store_path.is_empty() { + let resolved = policy.resolve_artifact_path(&artifact.path)?; + let ingested = self.store.ingest_artifact( + run_id, + &update.task_id, + &resolved, + &artifact.path, + MAX_ARTIFACT_BYTES, + now, + )?; + if ingested.digest != artifact.digest + || ingested.size_bytes != artifact.size_bytes + { + return Err(RuntimeError::InvalidState(format!( + "legacy artifact `{}` changed after analysis; expected {} bytes and `{}`, found {} bytes and `{}`", + artifact.path, + artifact.size_bytes, + artifact.digest, + ingested.size_bytes, + ingested.digest + ))); + } + *artifact = ingested; + } + } + } + let upgrade_id = self.ids.next_id("upgrade"); + let trace_id = self.ids.next_id("trace"); + let analysis_value = serde_json::to_value(&analysis_before)?; + self.store.apply_legacy_run_upgrade( + &upgrade_id, + run_id, + &analysis_value, + &updates, + now, + &trace_id, + )?; + let upgraded_tasks = updates + .iter() + .map(|update| update.task_id.clone()) + .collect(); + let analysis_after = self.analyze_legacy_run(run_id)?; + Ok(LegacyRunUpgradeResult { + upgrade_id, + run_id: run_id.to_owned(), + upgraded_tasks, + analysis_before, + analysis_after, + }) + } + + pub fn reconcile_effect( + &self, + input: EffectReconciliationInput, + ) -> Result { + let effect = self.store.load_effect(&input.effect_id)?; + if input.evidence.is_null() { + return Err(RuntimeError::InvalidState( + "effect reconciliation requires evidence".to_owned(), + )); + } + if let (Some(result), Some(schema)) = (&input.result, &input.result_schema) { + validate_output_contract(schema, result).map_err(|message| { + RuntimeError::InvalidState(format!( + "reconciled result failed the supplied schema: {message}" + )) + })?; + } + if input.status == ReconciliationStatus::Applied { + let result = input.result.as_ref().ok_or_else(|| { + RuntimeError::InvalidState( + "an applied reconciliation requires --result-file".to_owned(), + ) + })?; + if effect.request.effect_class == EffectClass::Model { + serde_json::from_value::(result.clone()).map_err(|error| { + RuntimeError::InvalidState(format!( + "reconciled model result is not a provider response: {error}" + )) + })?; + } + if let Some(tool_id) = effect.request.operation.strip_prefix("tool.") { + let tool = self.registry.tools.get(tool_id).ok_or_else(|| { + RuntimeError::InvalidState(format!( + "tool-specific reconciliation requires registered tool `{tool_id}`" + )) + })?; + tool.contract().validate_output(result)?; + } + } + if let Some(hook) = self + .registry + .reconciliation_hooks + .get(&effect.request.operation) + { + hook.validate(&effect, &input.evidence, input.result.as_ref()) + .map_err(|message| { + RuntimeError::InvalidState(format!( + "reconciliation hook for `{}` rejected the evidence: {message}", + effect.request.operation + )) + })?; + } + + let run = self.store.load_run(&effect.request.run_id)?; + let workflow: Workflow = serde_json::from_value(run.workflow)?; + let base_path = run + .base_path + .as_deref() + .map(PathBuf::from) + .unwrap_or_else(|| self.base_path.clone()); + let policy = PolicyEngine::new(workflow.spec.policy, &base_path)?; + let trace_id = self.ids.next_id("trace"); + let decision = policy.decide(&PolicyContext { + run_id: effect.request.run_id.clone(), + trace_id: trace_id.clone(), + task_id: effect.request.task_id.clone(), + agent: None, + tool: effect.request.operation.clone(), + capability: "effect_reconciliation".to_owned(), + effect_class: effect.request.effect_class, + risk: effect.request.risk, + resource: Some(effect.request.id.clone()), + provider: (effect.request.effect_class == EffectClass::Model) + .then(|| effect.request.operation.clone()), + input: serde_json::json!({ + "status": input.status, + "hasResult": input.result.is_some(), + "compensationEffectId": input.compensation_effect_id, + }), + interactive: input.approved, + }); + let authorization = match decision { + PolicyDecision::Deny { reason } => { + return Err(RuntimeError::InvalidState(format!( + "policy denied effect reconciliation: {reason}" + ))); + } + PolicyDecision::RequireApproval { reason } if !input.approved => { + return Err(RuntimeError::InvalidState(format!( + "effect reconciliation requires explicit --approved confirmation: {reason}" + ))); + } + PolicyDecision::RequireApproval { reason } => serde_json::json!({ + "kind": "explicit_operator_approval", + "actor": input.actor, + "reason": reason, + }), + PolicyDecision::Allow { reason } => serde_json::json!({ + "kind": "policy_allow", + "reason": reason, + "explicitApproval": input.approved, + }), + }; + let request = EffectReconciliationRequest { + reconciliation_id: self.ids.next_id("reconciliation"), + effect_id: input.effect_id, + status: input.status, + actor: input.actor, + reason: input.reason, + evidence: input.evidence, + result: input.result, + result_schema: input.result_schema, + authorization, + compensation_effect_id: input.compensation_effect_id, + trace_id, + }; + self.store + .reconcile_effect(&request, self.clock.now()) + .map_err(RuntimeError::from) + } + + fn analyze_legacy_run_internal( + &self, + run_id: &str, + ) -> Result<(LegacyRunUpgradeAnalysis, Vec), RuntimeError> { + let source = self.store.load_run(run_id)?; + if !source.state.is_terminal() { + return Err(RuntimeError::InvalidState(format!( + "legacy run analysis requires a terminal run, found {:?}", + source.state + ))); + } + let workflow: Workflow = serde_json::from_value(source.workflow.clone())?; + let base_path = source + .base_path + .as_deref() + .map(PathBuf::from) + .unwrap_or_else(|| self.base_path.clone()); + let policy = PolicyEngine::new(workflow.spec.policy.clone(), &base_path)?; + let inputs = source + .inputs + .as_object() + .ok_or_else(|| RuntimeError::InvalidState("run inputs must be an object".to_owned()))?; + let task_records = self + .store + .list_tasks(run_id)? + .into_iter() + .map(|task| (task.task_id.clone(), task)) + .collect::>(); + let effects = self.store.list_effects(run_id)?; + let checkpoints = self.store.checkpoints(run_id)?; + let outputs = task_records + .iter() + .filter_map(|(task_id, task)| { + task.output + .as_ref() + .map(|output| (task_id.clone(), output.clone())) + }) + .collect::>(); + let mut tasks = Vec::new(); + let mut updates = Vec::new(); + let mut unavailable = Vec::new(); + + for task_id in &source.plan.order { + let task = task_records.get(task_id).ok_or_else(|| { + RuntimeError::InvalidState(format!( + "compiled legacy task `{task_id}` has no durable task row" + )) + })?; + if task.state != TaskState::Succeeded { + unavailable.push(task_id.clone()); + tasks.push(LegacyTaskUpgradeAnalysis { + task_id: task_id.clone(), + state: task.state, + already_current: false, + upgradeable: false, + confidence: "unavailable".to_owned(), + reasons: vec![format!( + "task is {:?}; only successful task results can be upgraded for reuse", + task.state + )], + provenance: BTreeMap::new(), + proposed_metadata: None, + }); + continue; + } + if task.metadata_version == Some(TASK_METADATA_VERSION) { + tasks.push(LegacyTaskUpgradeAnalysis { + task_id: task_id.clone(), + state: task.state, + already_current: true, + upgradeable: false, + confidence: "already_current".to_owned(), + reasons: Vec::new(), + provenance: BTreeMap::new(), + proposed_metadata: None, + }); + continue; + } + if task.metadata_version.is_some() { + unavailable.push(task_id.clone()); + tasks.push(LegacyTaskUpgradeAnalysis { + task_id: task_id.clone(), + state: task.state, + already_current: false, + upgradeable: false, + confidence: "unsupported".to_owned(), + reasons: vec![format!( + "task metadata version {:?} is not supported by upgrader version {TASK_METADATA_VERSION}", + task.metadata_version + )], + provenance: BTreeMap::new(), + proposed_metadata: None, + }); + continue; + } + + let compiled = source.plan.tasks.get(task_id).ok_or_else(|| { + RuntimeError::InvalidState(format!("compiled task `{task_id}` disappeared")) + })?; + let task_effects = effects + .iter() + .filter(|effect| effect.request.task_id == *task_id) + .cloned() + .collect::>(); + let (metadata, provenance, reasons) = derive_legacy_task_metadata( + &workflow, + compiled, + task, + &policy, + inputs, + &outputs, + &task_effects, + &checkpoints, + ); + let upgradeable = metadata.is_some(); + if !upgradeable { + unavailable.push(task_id.clone()); + } + if let Some(metadata) = &metadata { + updates.push(LegacyTaskUpgrade { + task_id: task_id.clone(), + metadata: metadata.clone(), + provenance: serde_json::json!({ + "formatVersion": 1, + "confidence": "proven", + "fields": provenance, + }), + }); + } + tasks.push(LegacyTaskUpgradeAnalysis { + task_id: task_id.clone(), + state: task.state, + already_current: false, + upgradeable, + confidence: if upgradeable { "proven" } else { "unavailable" }.to_owned(), + reasons, + provenance, + proposed_metadata: metadata, + }); + } + + let recommended_repair_roots = earliest_safe_repair_roots(&source.plan, &unavailable); + let upgradeable_tasks = updates + .iter() + .map(|update| update.task_id.clone()) + .collect::>(); + let legacy_successes = tasks + .iter() + .filter(|task| task.state == TaskState::Succeeded && !task.already_current) + .count(); + let unavailable_successes = tasks + .iter() + .filter(|task| { + task.state == TaskState::Succeeded && !task.already_current && !task.upgradeable + }) + .count(); + let already_current = legacy_successes == 0; + Ok(( + LegacyRunUpgradeAnalysis { + api_version: LEGACY_UPGRADE_ANALYSIS_VERSION.to_owned(), + run_id: run_id.to_owned(), + database_schema_version: self.store.schema_version(), + terminal: true, + fully_upgradeable: unavailable_successes == 0, + already_current, + upgradeable_tasks, + unavailable_tasks: unavailable, + recommended_repair_roots, + tasks, + }, + updates, + )) + } + pub async fn fork( &self, source_run_id: &str, @@ -823,7 +1331,10 @@ impl Runtime { .iter() .filter(|effect| rerun.contains(&effect.request.task_id)) { - if repair_effect_is_unsafe(effect) { + let reconciliation = self + .store + .latest_effect_reconciliation(&effect.request.id)?; + if repair_effect_is_unsafe(effect, reconciliation.as_ref()) { let task_id = effect.request.task_id.clone(); blocks.push(repair_block( &task_id, @@ -953,9 +1464,9 @@ impl Runtime { blocked( "legacy_task_metadata", format!( - "task `{task_id}` predates repair metadata version {TASK_METADATA_VERSION}; choose it as an earlier repair root or perform a full fork" + "task `{task_id}` predates repair metadata version {TASK_METADATA_VERSION}; run `agentctl runs analyze`/`runs upgrade`, or choose the reported safe repair root" ), - true, + false, &mut blocks, &mut blocked_task_ids, ); @@ -1028,7 +1539,7 @@ impl Runtime { )); continue; } - match unresolved_reuse_effects(source_task, &source_effects) { + match unresolved_reuse_effects(&self.store, source_task, &source_effects) { Ok(effect_ids) if !effect_ids.is_empty() => { blocked( "unresolved_reused_effect", @@ -1385,16 +1896,21 @@ impl Runtime { .filter(|effect| rerun.contains(&effect.task) && effect.approval_possible) .map(|effect| format!("{}:{}", effect.task, effect.operation)) .collect::>(); - let uncertain_source_effects = source_effects - .iter() - .filter(|effect| { - rerun.contains(&effect.request.task_id) - && matches!( - effect.status, - EffectStatus::Started | EffectStatus::Uncertain - ) - }) - .count(); + let mut uncertain_source_effects = 0; + for effect in &source_effects { + if rerun.contains(&effect.request.task_id) + && matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) + && self + .store + .latest_effect_reconciliation(&effect.request.id)? + .is_none() + { + uncertain_source_effects += 1; + } + } let compatible = blocks.is_empty(); Ok(RepairPlan { api_version: REPAIR_PLAN_VERSION.to_owned(), @@ -3100,6 +3616,27 @@ impl Runtime { ) -> Result { match self.store.load_effect(&request.id) { Ok(record) => { + if let Some(reconciliation) = + self.store.latest_effect_reconciliation(&request.id)? + { + return match reconciliation.status { + ReconciliationStatus::Applied => reconciliation + .result + .map(PreparedEffect::Recorded) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "applied reconciliation for effect `{}` has no result", + request.id + )) + }), + ReconciliationStatus::NotApplied | ReconciliationStatus::Compensated => { + Err(RuntimeError::InvalidState(format!( + "effect `{}` requires a fresh task attempt after {:?} reconciliation", + request.id, reconciliation.status + ))) + } + }; + } return match record.status { EffectStatus::Succeeded if record.confirmed => { record.result.map(PreparedEffect::Recorded).ok_or_else(|| { @@ -3317,7 +3854,10 @@ fn blocked_task_plan( } } -fn repair_effect_is_unsafe(effect: &EffectRecord) -> bool { +fn repair_effect_is_unsafe( + effect: &EffectRecord, + reconciliation: Option<&EffectReconciliationRecord>, +) -> bool { let potentially_mutating = matches!( effect.request.effect_class, EffectClass::WorkspaceMutate @@ -3326,6 +3866,18 @@ fn repair_effect_is_unsafe(effect: &EffectRecord) -> bool { | EffectClass::Network | EffectClass::RemoteAgent ); + if !potentially_mutating { + return false; + } + if let Some(reconciliation) = reconciliation { + return match reconciliation.status { + ReconciliationStatus::NotApplied | ReconciliationStatus::Compensated => false, + ReconciliationStatus::Applied => !matches!( + effect.request.idempotency, + Idempotency::Pure | Idempotency::Idempotent | Idempotency::Keyed + ), + }; + } potentially_mutating && (matches!( effect.status, @@ -3338,21 +3890,33 @@ fn repair_effect_is_unsafe(effect: &EffectRecord) -> bool { } fn unresolved_reuse_effects( + store: &SqliteStore, task: &TaskRecord, source_effects: &[EffectRecord], ) -> Result, String> { if task.disposition != TaskDisposition::Reused { - return Ok(source_effects + let mut unresolved = Vec::new(); + for effect in source_effects .iter() - .filter(|effect| { - effect.request.task_id == task.task_id - && matches!( - effect.status, - EffectStatus::Started | EffectStatus::Uncertain - ) - }) - .map(|effect| effect.request.id.clone()) - .collect()); + .filter(|effect| effect.request.task_id == task.task_id) + { + let reconciliation = store + .latest_effect_reconciliation(&effect.request.id) + .map_err(|error| error.to_string())?; + if reconciliation + .as_ref() + .is_some_and(|record| record.status == ReconciliationStatus::Compensated) + || (matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) && !reconciliation + .as_ref() + .is_some_and(|record| record.status == ReconciliationStatus::Applied)) + { + unresolved.push(effect.request.id.clone()); + } + } + return Ok(unresolved); } let summaries = task @@ -3371,8 +3935,25 @@ fn unresolved_reuse_effects( .get("status") .and_then(Value::as_str) .ok_or_else(|| format!("source effect `{effect_id}` has no status"))?; + let reconciliation = store + .latest_effect_reconciliation(effect_id) + .map_err(|error| error.to_string())?; + if reconciliation + .as_ref() + .is_some_and(|record| record.status == ReconciliationStatus::Compensated) + { + unresolved.push(effect_id.to_owned()); + continue; + } match status { - "started" | "uncertain" => unresolved.push(effect_id.to_owned()), + "started" | "uncertain" + if !reconciliation + .as_ref() + .is_some_and(|record| record.status == ReconciliationStatus::Applied) => + { + unresolved.push(effect_id.to_owned()); + } + "started" | "uncertain" => {} "requested" | "waiting_for_approval" | "succeeded" | "failed" | "cancelled" => {} other => { return Err(format!( @@ -3607,8 +4188,418 @@ fn canonical_json(value: &Value) -> Value { } } -fn read_bounded_text_sync(path: &Path) -> Result { - use std::io::Read as _; +#[allow(clippy::too_many_arguments)] +fn derive_legacy_task_metadata( + workflow: &Workflow, + compiled: &agentctl_core::CompiledTask, + task: &TaskRecord, + policy: &PolicyEngine, + inputs: &serde_json::Map, + outputs: &BTreeMap, + effects: &[EffectRecord], + checkpoints: &[CheckpointRecord], +) -> ( + Option, + BTreeMap, + Vec, +) { + let mut provenance = BTreeMap::new(); + let mut reasons = Vec::new(); + + let definition_fingerprint = match task_definition_fingerprint( + workflow, + compiled, + policy, + Some(effects), + ) { + Ok(fingerprint) => { + provenance.insert( + "definitionFingerprint".to_owned(), + "stored workflow, compiled plan, policy, tools, provider, and recorded instruction read" + .to_owned(), + ); + Some(fingerprint) + } + Err(error) => { + reasons.push(format!("definition fingerprint cannot be proven: {error}")); + None + } + }; + + let contract = task_output_schema(workflow, compiled).unwrap_or_else(|| serde_json::json!({})); + let output_contract_fingerprint = match versioned_json_digest(&contract) { + Ok(fingerprint) => { + provenance.insert( + "outputContractFingerprint".to_owned(), + "stored workflow task/agent output schema".to_owned(), + ); + Some(fingerprint) + } + Err(error) => { + reasons.push(format!("output contract cannot be hashed: {error}")); + None + } + }; + + let output_digest = match task.output.as_ref() { + Some(output) => { + if let Err(error) = validate_output_contract(&contract, output) { + reasons.push(format!( + "stored successful output does not satisfy its contract: {error}" + )); + None + } else { + match versioned_json_digest(output) { + Ok(digest) => { + provenance.insert( + "outputDigest".to_owned(), + "stored successful task output".to_owned(), + ); + Some(digest) + } + Err(error) => { + reasons.push(format!("stored output cannot be hashed: {error}")); + None + } + } + } + } + None => { + reasons.push("successful task has no stored output".to_owned()); + None + } + }; + + let boundary = match legacy_checkpoint_boundary(checkpoints, &task.task_id) { + Ok(boundary) => { + provenance.insert( + "checkpointBoundary".to_owned(), + format!( + "checksummed checkpoints {} and {} around the successful transition", + boundary.0, boundary.1 + ), + ); + Some(boundary) + } + Err(error) => { + reasons.push(error); + None + } + }; + let input_digest = boundary.as_ref().and_then(|boundary| { + match resolved_input_digest(inputs, &boundary.2, outputs, compiled) { + Ok(digest) => { + provenance.insert( + "inputDigest".to_owned(), + "stored inputs, dependency outputs, task variables, and pre-task checkpoint memory" + .to_owned(), + ); + Some(digest) + } + Err(error) => { + reasons.push(format!("resolved input boundary cannot be proven: {error}")); + None + } + } + }); + let state_delta = + boundary.as_ref().and_then( + |boundary| match state_delta(&boundary.2, Some(&boundary.3)) { + Ok(delta) => { + provenance.insert( + "stateDelta".to_owned(), + "difference between checksummed pre/post-task working memory".to_owned(), + ); + Some(delta) + } + Err(error) => { + reasons.push(format!("state delta cannot be reconstructed: {error}")); + None + } + }, + ); + let state_delta_digest = + state_delta + .as_ref() + .and_then(|delta| match versioned_json_digest(delta) { + Ok(digest) => Some(digest), + Err(error) => { + reasons.push(format!("state delta cannot be hashed: {error}")); + None + } + }); + + let artifact_manifest = match analyze_legacy_artifacts(task, effects, policy) { + Ok(artifacts) => { + provenance.insert( + "artifactManifest".to_owned(), + if task.artifact_manifest.is_empty() { + "confirmed workspace-mutation effects plus current policy-authorized bytes" + .to_owned() + } else { + "legacy manifest identity plus current policy-authorized bytes".to_owned() + }, + ); + Some(artifacts) + } + Err(error) => { + reasons.push(format!("artifact manifest cannot be proven: {error}")); + None + } + }; + + let metadata = match ( + definition_fingerprint, + input_digest, + output_contract_fingerprint, + output_digest, + state_delta, + state_delta_digest, + artifact_manifest, + ) { + ( + Some(definition_fingerprint), + Some(input_digest), + Some(output_contract_fingerprint), + Some(output_digest), + Some(state_delta), + Some(state_delta_digest), + Some(artifact_manifest), + ) if reasons.is_empty() => Some(TaskCompletionMetadata { + execution: TaskExecutionMetadata { + metadata_version: TASK_METADATA_VERSION, + definition_fingerprint, + input_digest, + output_contract_fingerprint, + }, + output_digest, + state_delta, + state_delta_digest, + artifact_manifest, + }), + _ => None, + }; + (metadata, provenance, reasons) +} + +fn legacy_checkpoint_boundary( + checkpoints: &[CheckpointRecord], + task_id: &str, +) -> Result<(i64, i64, Value, Value), String> { + for pair in checkpoints.windows(2) { + let before_state = checkpoint_task_state(&pair[0].state, task_id); + let after_state = checkpoint_task_state(&pair[1].state, task_id); + if before_state.as_deref() != Some("succeeded") + && after_state.as_deref() == Some("succeeded") + { + let before_memory = pair[0] + .state + .get("workingMemory") + .filter(|memory| memory.is_object()) + .cloned() + .ok_or_else(|| { + format!( + "checkpoint {} has no object workingMemory before task `{task_id}`", + pair[0].sequence + ) + })?; + let after_memory = pair[1] + .state + .get("workingMemory") + .filter(|memory| memory.is_object()) + .cloned() + .ok_or_else(|| { + format!( + "checkpoint {} has no object workingMemory after task `{task_id}`", + pair[1].sequence + ) + })?; + return Ok(( + pair[0].sequence, + pair[1].sequence, + before_memory, + after_memory, + )); + } + } + Err(format!( + "no consecutive checksummed checkpoints prove the successful transition for task `{task_id}`" + )) +} + +fn checkpoint_task_state(checkpoint: &Value, task_id: &str) -> Option { + checkpoint + .get("tasks") + .and_then(Value::as_array)? + .iter() + .find(|task| task.get("taskId").and_then(Value::as_str) == Some(task_id))? + .get("state") + .and_then(Value::as_str) + .map(ToOwned::to_owned) +} + +fn analyze_legacy_artifacts( + task: &TaskRecord, + effects: &[EffectRecord], + policy: &PolicyEngine, +) -> Result, String> { + let mut expected = task + .artifact_manifest + .iter() + .map(|artifact| (artifact.path.clone(), Some(artifact))) + .collect::>(); + let mutations = effects + .iter() + .filter(|effect| effect.request.effect_class == EffectClass::WorkspaceMutate) + .collect::>(); + if expected.is_empty() { + for effect in &mutations { + if effect.status != EffectStatus::Succeeded || !effect.confirmed { + return Err(format!( + "workspace mutation `{}` is not a confirmed success", + effect.request.id + )); + } + let result = effect.result.as_ref().ok_or_else(|| { + format!( + "confirmed workspace mutation `{}` has no stored result", + effect.request.id + ) + })?; + let mut paths = BTreeSet::new(); + collect_result_paths(result, &mut paths); + if paths.is_empty() { + return Err(format!( + "confirmed workspace mutation `{}` has no stored output path", + effect.request.id + )); + } + for path in paths { + expected.insert(path, None); + } + } + } + + expected + .into_iter() + .map(|(path, legacy)| { + let resolved = policy + .resolve_artifact_path(&path) + .map_err(|error| error.to_string())?; + let (digest, size_bytes) = hash_bounded_artifact(&resolved)?; + if let Some(legacy) = legacy + && (legacy.digest != digest || legacy.size_bytes != size_bytes) + { + return Err(format!( + "legacy artifact `{path}` identity changed: expected {} bytes and `{}`, found {} bytes and `{digest}`", + legacy.size_bytes, legacy.digest, size_bytes + )); + } + let logical_name = Path::new(&path) + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| format!("artifact `{path}` has no valid logical name"))?; + Ok(ArtifactRecord { + path: path.clone(), + digest, + size_bytes, + media_type: legacy + .map(|artifact| artifact.media_type.clone()) + .filter(|media_type| !media_type.is_empty()) + .unwrap_or_else(|| legacy_media_type(Path::new(&path)).to_owned()), + logical_name: legacy + .map(|artifact| artifact.logical_name.clone()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| logical_name.to_owned()), + store_path: legacy + .map(|artifact| artifact.store_path.clone()) + .unwrap_or_default(), + }) + }) + .collect() +} + +fn hash_bounded_artifact(path: &Path) -> Result<(String, u64), String> { + use std::io::Read as _; + + let metadata = std::fs::symlink_metadata(path).map_err(|error| error.to_string())?; + if !metadata.file_type().is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + if metadata.len() > MAX_ARTIFACT_BYTES { + return Err(format!( + "{} exceeds the {} byte artifact limit", + path.display(), + MAX_ARTIFACT_BYTES + )); + } + let file = std::fs::File::open(path).map_err(|error| error.to_string())?; + let mut reader = file.take(MAX_ARTIFACT_BYTES + 1); + let mut bytes = Vec::new(); + reader + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if u64::try_from(bytes.len()).unwrap_or(u64::MAX) > MAX_ARTIFACT_BYTES { + return Err(format!( + "{} changed while reading and exceeded the {} byte artifact limit", + path.display(), + MAX_ARTIFACT_BYTES + )); + } + Ok(( + format!("sha256:{}", digest(&bytes)), + u64::try_from(bytes.len()).unwrap_or(u64::MAX), + )) +} + +fn legacy_media_type(path: &Path) -> &'static str { + match path.extension().and_then(|extension| extension.to_str()) { + Some("json") => "application/json", + Some("yaml" | "yml") => "application/yaml", + Some("md") => "text/markdown", + Some("txt" | "log" | "csv") => "text/plain", + _ => "application/octet-stream", + } +} + +fn earliest_safe_repair_roots(plan: &CompiledPlan, unavailable: &[String]) -> Vec { + let unavailable = unavailable.iter().cloned().collect::>(); + if unavailable.is_empty() { + return Vec::new(); + } + let earliest = plan + .order + .iter() + .position(|task_id| unavailable.contains(task_id)) + .unwrap_or(plan.order.len()); + let mut covered = BTreeSet::new(); + let mut roots = Vec::new(); + for task_id in plan.order.iter().skip(earliest) { + if !covered.contains(task_id) { + roots.push(task_id.clone()); + covered.insert(task_id.clone()); + loop { + let before = covered.len(); + for candidate in &plan.order { + if plan + .tasks + .get(candidate) + .is_some_and(|task| task.needs.iter().any(|need| covered.contains(need))) + { + covered.insert(candidate.clone()); + } + } + if covered.len() == before { + break; + } + } + } + } + roots +} + +fn read_bounded_text_sync(path: &Path) -> Result { + use std::io::Read as _; let file = std::fs::File::open(path)?; let mut reader = file.take(MAX_WORKSPACE_FILE_BYTES + 1); @@ -4369,6 +5360,19 @@ mod tests { } } + struct RejectReconciliationHook; + + impl EffectReconciliationHook for RejectReconciliationHook { + fn validate( + &self, + _effect: &EffectRecord, + _evidence: &Value, + _result: Option<&Value>, + ) -> Result<(), String> { + Err("external verifier did not confirm the record".to_owned()) + } + } + fn compile_fixture(source: &str) -> (Workflow, CompiledPlan) { let workflow = parse_workflow(source, "fixture.yaml") .expect("parse fixture") @@ -4455,6 +5459,405 @@ spec: .with_ids(Arc::new(SequenceIds::default())) } + fn running_memory_effect( + store: &SqliteStore, + base: &Path, + run_id: &str, + uncertain: bool, + ) -> EffectRequest { + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: reconciled-resume } +spec: + policy: { approval: never } + memory: + working: { value: old } + actions: + remember: { kind: builtin.memory.write } + tasks: + - id: remember + uses: action:remember + with: { key: value, value: new } +"#; + let (workflow, plan) = compile_fixture(source); + store + .create_run( + run_id, + API_VERSION, + &serde_json::to_value(workflow).expect("workflow"), + &plan, + &serde_json::json!({}), + &serde_json::json!({"value": "old"}), + RunMode::Execute, + None, + base, + FixedClock.now(), + "trace-source", + ) + .expect("create interrupted run"); + store + .transition_task( + run_id, + "remember", + TaskState::Ready, + None, + None, + None, + FixedClock.now(), + "trace-source", + ) + .expect("ready"); + store + .transition_task( + run_id, + "remember", + TaskState::Running, + None, + None, + None, + FixedClock.now(), + "trace-source", + ) + .expect("running"); + let effect = EffectRequest::new( + run_id, + "remember", + 1, + 1, + "builtin.memory.write", + EffectClass::InternalState, + Risk::Low, + Idempotency::Keyed, + serde_json::json!({"key": "value", "value": "new"}), + "update transactional run working memory", + "trace-source", + ); + store + .record_effect_request(&effect, FixedClock.now()) + .expect("effect"); + store + .mark_effect_started(&effect.id, FixedClock.now()) + .expect("started"); + if uncertain { + store + .mark_effect_uncertain(&effect.id, "interrupted", FixedClock.now()) + .expect("uncertain"); + } else { + store + .complete_effect( + &effect.id, + Ok(&serde_json::json!({ + "status": "changed", + "changed": true, + "before": "old", + "after": "new", + "key": "value", + })), + FixedClock.now(), + ) + .expect("complete"); + } + effect + } + + #[tokio::test] + async fn applied_reconciliation_supplies_a_validated_result_to_resume() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let effect = running_memory_effect(&store, directory.path(), "resume-applied", true); + let runtime = runtime(store.clone(), directory.path()); + let result = serde_json::json!({ + "status": "changed", + "changed": true, + "before": "old", + "after": "new", + "key": "value", + }); + let reconciliation = runtime + .reconcile_effect(EffectReconciliationInput { + effect_id: effect.id.clone(), + status: ReconciliationStatus::Applied, + actor: "operator".to_owned(), + reason: "checkpoint confirms the memory write".to_owned(), + evidence: serde_json::json!({"checkpoint": "external-1"}), + result: Some(result), + result_schema: Some(serde_json::json!({ + "type": "object", + "required": ["status", "changed", "key"], + })), + compensation_effect_id: None, + approved: false, + }) + .expect("reconcile applied"); + assert_eq!(reconciliation.status, ReconciliationStatus::Applied); + let outcome = runtime + .resume( + "resume-applied", + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("resume"); + assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!( + store + .load_run("resume-applied") + .expect("run") + .working_memory["value"], + "new" + ); + assert_eq!( + store.list_effects("resume-applied").expect("effects").len(), + 1 + ); + assert_eq!( + store.load_effect(&effect.id).expect("source effect").status, + EffectStatus::Uncertain + ); + } + + #[tokio::test] + async fn not_applied_reconciliation_resumes_with_a_fresh_task_and_effect_attempt() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let effect = running_memory_effect(&store, directory.path(), "resume-not-applied", true); + let runtime = runtime(store.clone(), directory.path()); + runtime + .reconcile_effect(EffectReconciliationInput { + effect_id: effect.id.clone(), + status: ReconciliationStatus::NotApplied, + actor: "operator".to_owned(), + reason: "checkpoint confirms no write".to_owned(), + evidence: serde_json::json!({"checkpoint": "external-2"}), + result: None, + result_schema: None, + compensation_effect_id: None, + approved: false, + }) + .expect("reconcile not applied"); + let outcome = runtime + .resume( + "resume-not-applied", + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("resume"); + assert_eq!(outcome.state, RunState::Succeeded); + let task = store + .list_tasks("resume-not-applied") + .expect("tasks") + .remove(0); + assert_eq!(task.attempt, 2); + let effects = store.list_effects("resume-not-applied").expect("effects"); + assert_eq!(effects.len(), 2); + assert_eq!(effects[0].request.id, effect.id); + assert_eq!(effects[0].status, EffectStatus::Uncertain); + assert_eq!(effects[1].status, EffectStatus::Succeeded); + assert_ne!(effects[0].request.id, effects[1].request.id); + } + + #[tokio::test] + async fn compensated_reconciliation_resumes_a_completed_effect_with_a_fresh_attempt() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let effect = running_memory_effect(&store, directory.path(), "resume-compensated", false); + let compensation = EffectRequest::new( + "resume-compensated", + "remember", + 1, + 2, + "builtin.memory.restore", + EffectClass::InternalState, + Risk::Low, + Idempotency::Keyed, + serde_json::json!({"key": "value", "value": "old"}), + "restore transactional run working memory", + "trace-source", + ); + store + .record_effect_request(&compensation, FixedClock.now()) + .expect("compensation"); + store + .mark_effect_started(&compensation.id, FixedClock.now()) + .expect("compensation started"); + store + .complete_effect( + &compensation.id, + Ok(&serde_json::json!({"restored": true})), + FixedClock.now(), + ) + .expect("compensation complete"); + let runtime = runtime(store.clone(), directory.path()); + runtime + .reconcile_effect(EffectReconciliationInput { + effect_id: effect.id.clone(), + status: ReconciliationStatus::Compensated, + actor: "operator".to_owned(), + reason: "the state write was reversed".to_owned(), + evidence: serde_json::json!({"checkpoint": "external-3"}), + result: None, + result_schema: None, + compensation_effect_id: Some(compensation.id), + approved: false, + }) + .expect("reconcile compensated"); + let outcome = runtime + .resume( + "resume-compensated", + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("resume"); + assert_eq!(outcome.state, RunState::Succeeded); + let task = store + .list_tasks("resume-compensated") + .expect("tasks") + .remove(0); + assert_eq!(task.attempt, 2); + let effects = store.list_effects("resume-compensated").expect("effects"); + assert_eq!(effects.len(), 3); + assert_eq!(effects[0].status, EffectStatus::Succeeded); + assert_eq!(effects[2].status, EffectStatus::Succeeded); + assert_ne!(effects[0].request.id, effects[2].request.id); + } + + #[test] + fn reconciliation_enforces_tool_contract_hook_and_policy_approval() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: reconciliation-policy } +spec: + policy: + approval: high_risk + nonInteractive: pause + actions: + assign: { kind: builtin.assign } + tasks: + - { id: one, uses: "action:assign", with: { value: one } } +"#; + let (workflow, plan) = compile_fixture(source); + store + .create_run( + "reconciliation-policy", + API_VERSION, + &serde_json::to_value(workflow).expect("workflow"), + &plan, + &serde_json::json!({}), + &serde_json::json!({}), + RunMode::Execute, + None, + directory.path(), + FixedClock.now(), + "trace", + ) + .expect("run"); + let effect = EffectRequest::new( + "reconciliation-policy", + "one", + 1, + 1, + "tool.echo", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"text": "hello"}), + "external echo", + "trace", + ); + store + .record_effect_request(&effect, FixedClock.now()) + .expect("effect"); + store + .mark_effect_started(&effect.id, FixedClock.now()) + .expect("started"); + store + .mark_effect_uncertain(&effect.id, "unknown", FixedClock.now()) + .expect("uncertain"); + let runtime_instance = runtime(store.clone(), directory.path()).with_registry( + RuntimeRegistry::default().with_tool("echo", Arc::new(FixtureTool::new(false))), + ); + let base = EffectReconciliationInput { + effect_id: effect.id.clone(), + status: ReconciliationStatus::Applied, + actor: "operator".to_owned(), + reason: "external verifier confirms output".to_owned(), + evidence: serde_json::json!({"externalId": "echo-1"}), + result: Some(serde_json::json!({"text": "hello"})), + result_schema: None, + compensation_effect_id: None, + approved: false, + }; + let mut invalid = base.clone(); + invalid.result = Some(serde_json::json!({"wrong": true})); + assert!(matches!( + runtime_instance.reconcile_effect(invalid), + Err(RuntimeError::Tool(_)) + )); + assert!(runtime_instance.reconcile_effect(base.clone()).is_err()); + let mut approved = base; + approved.approved = true; + assert_eq!( + runtime_instance + .reconcile_effect(approved) + .expect("approved") + .status, + ReconciliationStatus::Applied + ); + + let hook_effect = EffectRequest::new( + "reconciliation-policy", + "one", + 1, + 2, + "external.verify", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"record": "x"}), + "verify record", + "trace", + ); + store + .record_effect_request(&hook_effect, FixedClock.now()) + .expect("hook effect"); + store + .mark_effect_started(&hook_effect.id, FixedClock.now()) + .expect("hook started"); + store + .mark_effect_uncertain(&hook_effect.id, "unknown", FixedClock.now()) + .expect("hook uncertain"); + let hooked = runtime(store.clone(), directory.path()).with_registry( + RuntimeRegistry::default() + .with_reconciliation_hook("external.verify", Arc::new(RejectReconciliationHook)), + ); + assert!(matches!( + hooked.reconcile_effect(EffectReconciliationInput { + effect_id: hook_effect.id.clone(), + status: ReconciliationStatus::Applied, + actor: "operator".to_owned(), + reason: "manual claim".to_owned(), + evidence: serde_json::json!({"claim": true}), + result: Some(serde_json::json!({"record": "x"})), + result_schema: None, + compensation_effect_id: None, + approved: true, + }), + Err(RuntimeError::InvalidState(message)) if message.contains("hook") + )); + assert!( + store + .effect_reconciliations(&hook_effect.id) + .expect("no hook reconciliation") + .is_empty() + ); + } + #[tokio::test] async fn selective_repair_reuses_upstream_agent_output_without_dispatch() { let directory = tempdir().expect("tempdir"); @@ -5454,6 +6857,172 @@ spec: assert_eq!(store.stats().expect("stats"), stats_before); } + #[tokio::test] + async fn legacy_analysis_upgrade_imports_artifacts_and_enables_selective_repair() { + let directory = tempdir().expect("tempdir"); + let database = directory.path().join("state").join("runtime.db"); + let store = SqliteStore::open(&database).expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: legacy-upgrade } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:write + with: { path: legacy.txt, content: durable } + - id: second + uses: action:assert + needs: [first] + with: { that: false } +"#; + let repaired_yaml = source_yaml.replace("with: { that: false }", "with: { that: true }"); + let (source_workflow, source_plan) = compile_fixture(source_yaml); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let raw = rusqlite::Connection::open(&database).expect("raw database"); + raw.execute( + "UPDATE task_states SET metadata_version = NULL, definition_fingerprint = NULL, input_digest = NULL, output_contract_fingerprint = NULL, output_digest = NULL, state_delta_json = NULL, state_delta_digest = NULL, artifact_manifest_json = NULL, reuse_decision_json = NULL WHERE run_id = ?1 AND task_id = 'first'", + [&source_run_id], + ) + .expect("simulate pre-v5 task"); + raw.execute( + "DELETE FROM artifact_refs WHERE run_id = ?1", + [&source_run_id], + ) + .expect("remove post-v5 references"); + raw.execute("DELETE FROM artifact_blobs", []) + .expect("remove post-v5 blob index"); + drop(raw); + + let stats_before = store.stats().expect("stats"); + let analysis = runtime + .analyze_legacy_run(&source_run_id) + .expect("dry-run analysis"); + assert_eq!(analysis.upgradeable_tasks, ["first"]); + assert_eq!(analysis.unavailable_tasks, ["second"]); + assert_eq!(analysis.recommended_repair_roots, ["second"]); + assert_eq!(store.stats().expect("stats"), stats_before); + assert_eq!( + store.list_tasks(&source_run_id).expect("tasks")[0].metadata_version, + None + ); + + let upgrade = runtime + .upgrade_legacy_run(&source_run_id) + .expect("legacy upgrade"); + assert_eq!(upgrade.upgraded_tasks, ["first"]); + assert!(upgrade.analysis_after.tasks[0].already_current); + let upgraded = store.list_tasks(&source_run_id).expect("upgraded tasks"); + assert_eq!(upgraded[0].metadata_version, Some(TASK_METADATA_VERSION)); + assert_eq!(upgraded[0].artifact_manifest.len(), 1); + assert_eq!(store.stats().expect("stats").run_upgrades, 1); + + std::fs::remove_file(directory.path().join("legacy.txt")).expect("remove workspace file"); + let (repaired_workflow, repaired_plan) = compile_fixture(&repaired_yaml); + let plan = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &upgrade.analysis_after.recommended_repair_roots, + false, + ) + .expect("repair plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + let outcome = runtime + .repair( + &repaired_workflow, + &repaired_plan, + plan, + Some("repair from proven legacy boundary"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("selective repair"); + assert_eq!(outcome.state, RunState::Succeeded); + let tasks = store.list_tasks(&outcome.run_id).expect("repair tasks"); + assert_eq!(tasks[0].disposition, TaskDisposition::Reused); + assert_eq!(tasks[1].disposition, TaskDisposition::Executed); + assert!(runtime.replay(&outcome.run_id).await.is_ok()); + } + + #[tokio::test] + async fn legacy_analysis_reports_the_earliest_safe_boundary_when_proof_is_missing() { + let directory = tempdir().expect("tempdir"); + let database = directory.path().join("runtime.db"); + let store = SqliteStore::open(&database).expect("store"); + let runtime = runtime(store, directory.path()); + let source_yaml = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: legacy-missing-proof } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - { id: first, uses: "action:assign", with: { value: first } } + - { id: second, uses: "action:assign", with: { value: second } } +"#; + let (workflow, plan) = compile_fixture(source_yaml); + let outcome = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("source"); + let raw = rusqlite::Connection::open(&database).expect("raw database"); + raw.execute( + "UPDATE task_states SET metadata_version = NULL, definition_fingerprint = NULL, input_digest = NULL, output_contract_fingerprint = NULL, output_digest = NULL, state_delta_json = NULL, state_delta_digest = NULL, artifact_manifest_json = NULL, reuse_decision_json = NULL WHERE run_id = ?1", + [&outcome.run_id], + ) + .expect("simulate legacy tasks"); + raw.execute( + "DELETE FROM checkpoints WHERE run_id = ?1 AND sequence > 1", + [&outcome.run_id], + ) + .expect("remove unavailable proof"); + drop(raw); + + let analysis = runtime + .analyze_legacy_run(&outcome.run_id) + .expect("analysis"); + assert!(!analysis.fully_upgradeable); + assert_eq!(analysis.unavailable_tasks, ["first", "second"]); + assert_eq!(analysis.recommended_repair_roots, ["first", "second"]); + assert!(analysis.tasks.iter().all(|task| { + task.proposed_metadata.is_none() + && task + .reasons + .iter() + .any(|reason| reason.contains("checksummed checkpoints")) + })); + } + #[tokio::test] async fn repair_blocks_missing_state_delta_and_tampered_reused_output_digest() { let directory = tempdir().expect("tempdir"); diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index 319a56c..3cea8ab 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -19,7 +19,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use thiserror::Error; -pub const DATABASE_SCHEMA_VERSION: u32 = 6; +pub const DATABASE_SCHEMA_VERSION: u32 = 8; pub const RUNTIME_STATE_VERSION: u32 = 1; pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; pub const AUDIT_EVENT_VERSION: u32 = 1; @@ -256,6 +256,43 @@ CREATE TABLE artifact_ingests ( CREATE INDEX idx_artifact_ingests_expiry ON artifact_ingests(expires_at); "#; +const MIGRATION_7: &str = r#" +CREATE TABLE run_upgrades ( + run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, + upgrade_id TEXT NOT NULL, + format_version INTEGER NOT NULL, + analysis_json TEXT NOT NULL, + upgraded_tasks_json TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (run_id, upgrade_id) +); +CREATE INDEX idx_run_upgrades_created ON run_upgrades(run_id, created_at); +"#; + +const MIGRATION_8: &str = r#" +CREATE TABLE effect_reconciliations ( + reconciliation_id TEXT PRIMARY KEY, + effect_id TEXT NOT NULL REFERENCES effects(effect_id), + run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, + format_version INTEGER NOT NULL, + status TEXT NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + evidence_json TEXT NOT NULL, + result_json TEXT, + result_schema_json TEXT, + authorization_json TEXT NOT NULL, + compensation_effect_id TEXT REFERENCES effects(effect_id), + supersedes_id TEXT REFERENCES effect_reconciliations(reconciliation_id), + trace_id TEXT NOT NULL, + created_at TEXT NOT NULL +); +CREATE INDEX idx_effect_reconciliations_effect_created + ON effect_reconciliations(effect_id, created_at, reconciliation_id); +CREATE INDEX idx_effect_reconciliations_run_created + ON effect_reconciliations(run_id, created_at, reconciliation_id); +"#; + #[derive(Clone)] pub struct SqliteStore { connection: Arc>, @@ -442,6 +479,58 @@ pub struct ReusedTaskMaterialization { pub reuse_decision: Value, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTaskUpgrade { + pub task_id: String, + pub metadata: TaskCompletionMetadata, + pub provenance: Value, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReconciliationStatus { + Applied, + NotApplied, + Compensated, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EffectReconciliationRequest { + pub reconciliation_id: String, + pub effect_id: String, + pub status: ReconciliationStatus, + pub actor: String, + pub reason: String, + pub evidence: Value, + pub result: Option, + pub result_schema: Option, + pub authorization: Value, + pub compensation_effect_id: Option, + pub trace_id: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EffectReconciliationRecord { + pub reconciliation_id: String, + pub effect_id: String, + pub run_id: String, + pub format_version: u32, + pub status: ReconciliationStatus, + pub actor: String, + pub reason: String, + pub evidence: Value, + pub result: Option, + pub result_schema: Option, + pub authorization: Value, + pub compensation_effect_id: Option, + pub supersedes_id: Option, + pub trace_id: String, + pub created_at: DateTime, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ApprovalRequest { @@ -537,6 +626,8 @@ pub struct DatabaseStats { pub artifact_blobs: i64, pub artifact_references: i64, pub artifact_ingests: i64, + pub run_upgrades: i64, + pub effect_reconciliations: i64, } impl SqliteStore { @@ -1327,6 +1418,111 @@ impl SqliteStore { Ok(()) } + pub fn apply_legacy_run_upgrade( + &self, + upgrade_id: &str, + run_id: &str, + analysis: &Value, + updates: &[LegacyTaskUpgrade], + now: DateTime, + trace_id: &str, + ) -> Result<(), StoreError> { + let _artifact_guard = self.artifact_lock.lock(); + let _artifact_file_lock = self.artifact_store.lock_exclusive()?; + for update in updates { + verify_artifact_manifest( + self.artifact_store.as_ref(), + &update.metadata.artifact_manifest, + )?; + } + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let run_state: String = transaction + .query_row( + "SELECT state FROM runs WHERE run_id = ?1", + [run_id], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| StoreError::RunNotFound(run_id.to_owned()))?; + let run_state: RunState = decode_enum(&run_state, "run.state")?; + if !run_state.is_terminal() { + return Err(StoreError::Incompatible(format!( + "legacy run upgrade requires a terminal run, found {run_state:?}" + ))); + } + for update in updates { + let changed = transaction.execute( + "UPDATE task_states SET metadata_version = ?3, definition_fingerprint = ?4, input_digest = ?5, output_contract_fingerprint = ?6, output_digest = ?7, state_delta_json = ?8, state_delta_digest = ?9, artifact_manifest_json = ?10, reuse_decision_json = ?11, updated_at = ?12 WHERE run_id = ?1 AND task_id = ?2 AND state = ?13 AND metadata_version IS NULL", + params![ + run_id, + update.task_id, + update.metadata.execution.metadata_version, + update.metadata.execution.definition_fingerprint, + update.metadata.execution.input_digest, + update.metadata.execution.output_contract_fingerprint, + update.metadata.output_digest, + encode(&update.metadata.state_delta)?, + update.metadata.state_delta_digest, + encode(&update.metadata.artifact_manifest)?, + encode(&serde_json::json!({"legacyUpgrade": update.provenance}))?, + now.to_rfc3339(), + encode_enum(TaskState::Succeeded)?, + ], + )?; + if changed != 1 { + return Err(StoreError::Incompatible(format!( + "legacy task `{}` is missing, not successful, or already upgraded", + update.task_id + ))); + } + record_artifact_references_tx( + &transaction, + run_id, + &update.task_id, + &update.metadata.artifact_manifest, + None, + None, + now, + )?; + transaction.execute( + "DELETE FROM artifact_ingests WHERE run_id = ?1 AND task_id = ?2", + params![run_id, update.task_id], + )?; + } + let upgraded_tasks = updates + .iter() + .map(|update| update.task_id.as_str()) + .collect::>(); + transaction.execute( + "INSERT INTO run_upgrades (run_id, upgrade_id, format_version, analysis_json, upgraded_tasks_json, created_at) VALUES (?1, ?2, 1, ?3, ?4, ?5)", + params![ + run_id, + upgrade_id, + encode(analysis)?, + encode(&upgraded_tasks)?, + now.to_rfc3339(), + ], + )?; + append_audit_tx( + &transaction, + run_id, + "run.legacy_upgraded", + None, + trace_id, + &serde_json::json!({ + "upgradeId": upgrade_id, + "formatVersion": 1, + "upgradedTasks": upgraded_tasks, + "analysis": analysis, + }), + now, + )?; + checkpoint_tx(&transaction, run_id, now)?; + transaction.commit()?; + Ok(()) + } + pub fn update_run_state( &self, run_id: &str, @@ -1549,7 +1745,13 @@ impl SqliteStore { pub fn unresolved_effects(&self, run_id: &str) -> Result, StoreError> { let connection = self.connection.lock(); let mut statement = connection.prepare( - "SELECT effect_id FROM effects WHERE run_id = ?1 AND status IN ('started', 'uncertain') ORDER BY rowid", + "SELECT e.effect_id FROM effects e + WHERE e.run_id = ?1 + AND e.status IN ('started', 'uncertain') + AND NOT EXISTS ( + SELECT 1 FROM effect_reconciliations r WHERE r.effect_id = e.effect_id + ) + ORDER BY e.rowid", )?; statement .query_map([run_id], |row| row.get(0))? @@ -1557,60 +1759,285 @@ impl SqliteStore { .map_err(StoreError::from) } - pub fn reconcile_effect_not_applied( + pub fn reconcile_effect( &self, - effect_id: &str, - actor: &str, - reason: &str, + request: &EffectReconciliationRequest, now: DateTime, - ) -> Result<(), StoreError> { + ) -> Result { + if request.actor.trim().is_empty() || request.reason.trim().is_empty() { + return Err(StoreError::Incompatible( + "effect reconciliation requires a non-empty actor and reason".to_owned(), + )); + } + if request.status == ReconciliationStatus::Applied && request.result.is_none() { + return Err(StoreError::Incompatible( + "an applied reconciliation requires an externally confirmed result".to_owned(), + )); + } + if request.status == ReconciliationStatus::Compensated + && request.compensation_effect_id.is_none() + { + return Err(StoreError::Incompatible( + "a compensated reconciliation requires a linked compensation effect".to_owned(), + )); + } + let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let run_id: Option = transaction + let source: (String, String, bool) = transaction .query_row( - "SELECT run_id FROM effects WHERE effect_id = ?1 AND status IN (?2, ?3)", - params![ - effect_id, - encode_enum(EffectStatus::Started)?, - encode_enum(EffectStatus::Uncertain)?, - ], - |row| row.get(0), + "SELECT run_id, status, confirmed FROM effects WHERE effect_id = ?1", + [&request.effect_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) - .optional()?; - let Some(run_id) = run_id else { - return Err(StoreError::Incompatible(format!( - "effect `{effect_id}` is missing or is not uncertain" - ))); - }; + .optional()? + .ok_or_else(|| StoreError::EffectNotFound(request.effect_id.clone()))?; + let source_status: EffectStatus = decode_enum(&source.1, "effect.status")?; + let previous = transaction + .query_row( + "SELECT reconciliation_id, effect_id, run_id, format_version, status, actor, reason, evidence_json, result_json, result_schema_json, authorization_json, compensation_effect_id, supersedes_id, trace_id, created_at + FROM effect_reconciliations + WHERE effect_id = ?1 + ORDER BY created_at DESC, rowid DESC + LIMIT 1", + [&request.effect_id], + decode_reconciliation_row, + ) + .optional()? + .map(reconciliation_from_row) + .transpose()?; + + if let Some(previous) = &previous { + let allowed = previous.status == request.status + || (previous.status == ReconciliationStatus::Applied + && request.status == ReconciliationStatus::Compensated); + if !allowed { + return Err(StoreError::Incompatible(format!( + "effect `{}` is already reconciled as {:?}; contradictory {:?} reconciliation is forbidden", + request.effect_id, previous.status, request.status + ))); + } + } else { + let uncertain_source = matches!( + source_status, + EffectStatus::Started | EffectStatus::Uncertain + ); + let confirmed_source = source_status == EffectStatus::Succeeded && source.2; + let allowed = match request.status { + ReconciliationStatus::Applied | ReconciliationStatus::NotApplied => { + uncertain_source + } + ReconciliationStatus::Compensated => confirmed_source, + }; + if !allowed { + return Err(StoreError::Incompatible(format!( + "effect `{}` in state {:?} cannot be reconciled as {:?}", + request.effect_id, source_status, request.status + ))); + } + } + + if let Some(compensation_effect_id) = &request.compensation_effect_id { + if compensation_effect_id == &request.effect_id { + return Err(StoreError::Incompatible( + "an effect cannot compensate itself".to_owned(), + )); + } + let compensation: (String, String, bool) = transaction + .query_row( + "SELECT run_id, status, confirmed FROM effects WHERE effect_id = ?1", + [compensation_effect_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()? + .ok_or_else(|| StoreError::EffectNotFound(compensation_effect_id.clone()))?; + if compensation.0 != source.0 { + return Err(StoreError::Incompatible( + "compensation effect must belong to the same run".to_owned(), + )); + } + let compensation_status: EffectStatus = + decode_enum(&compensation.1, "compensation_effect.status")?; + let reconciled_applied = transaction + .query_row( + "SELECT status FROM effect_reconciliations WHERE effect_id = ?1 ORDER BY created_at DESC, rowid DESC LIMIT 1", + [compensation_effect_id], + |row| row.get::<_, String>(0), + ) + .optional()? + .map(|status| decode_enum::(&status, "compensation_reconciliation.status")) + .transpose()? + == Some(ReconciliationStatus::Applied); + if !(compensation_status == EffectStatus::Succeeded && compensation.2 + || reconciled_applied) + { + return Err(StoreError::Incompatible(format!( + "compensation effect `{compensation_effect_id}` is not confirmed applied" + ))); + } + } + + let supersedes_id = previous + .as_ref() + .map(|record| record.reconciliation_id.as_str()); transaction.execute( - "UPDATE effects SET status = ?2, error = ?3, completed_at = ?4, confirmed = 0 WHERE effect_id = ?1", + "INSERT INTO effect_reconciliations (reconciliation_id, effect_id, run_id, format_version, status, actor, reason, evidence_json, result_json, result_schema_json, authorization_json, compensation_effect_id, supersedes_id, trace_id, created_at) + VALUES (?1, ?2, ?3, 1, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", params![ - effect_id, - encode_enum(EffectStatus::Failed)?, - format!("operator reconciled as not applied: {reason}"), + request.reconciliation_id, + request.effect_id, + source.0, + encode_enum(request.status)?, + request.actor, + request.reason, + encode(&request.evidence)?, + request.result.as_ref().map(encode).transpose()?, + request.result_schema.as_ref().map(encode).transpose()?, + encode(&request.authorization)?, + request.compensation_effect_id, + supersedes_id, + request.trace_id, now.to_rfc3339(), ], )?; - transaction.execute( - "UPDATE tool_calls SET status = 'failed', completed_at = COALESCE(completed_at, ?2) WHERE run_id = ?1 AND effect_id = ?3 AND status IN ('started', 'uncertain')", - params![run_id, now.to_rfc3339(), effect_id], - )?; + let payload = serde_json::json!({ + "reconciliationId": request.reconciliation_id, + "effectId": request.effect_id, + "status": request.status, + "actor": request.actor, + "reason": request.reason, + "hasEvidence": !request.evidence.is_null(), + "hasResult": request.result.is_some(), + "compensationEffectId": request.compensation_effect_id, + "supersedesId": supersedes_id, + }); append_audit_tx( &transaction, - &run_id, - "effect.reconciled_not_applied", + &source.0, + "effect.reconciled", None, - "operator-reconciliation", + &request.trace_id, + &payload, + now, + )?; + append_trace_tx( + &transaction, + &source.0, + &request.trace_id, &serde_json::json!({ - "effectId": effect_id, - "actor": actor, - "reason": reason, - "outcome": "not_applied", + "spanKind": "effect", + "phase": "completed", + "name": "effect.reconcile", + "runId": source.0, + "traceId": request.trace_id, + "effectId": request.effect_id, + "timestamp": now, + "attributes": payload, }), now, )?; transaction.commit()?; - Ok(()) + Ok(EffectReconciliationRecord { + reconciliation_id: request.reconciliation_id.clone(), + effect_id: request.effect_id.clone(), + run_id: source.0, + format_version: 1, + status: request.status, + actor: request.actor.clone(), + reason: request.reason.clone(), + evidence: request.evidence.clone(), + result: request.result.clone(), + result_schema: request.result_schema.clone(), + authorization: request.authorization.clone(), + compensation_effect_id: request.compensation_effect_id.clone(), + supersedes_id: supersedes_id.map(ToOwned::to_owned), + trace_id: request.trace_id.clone(), + created_at: now, + }) + } + + pub fn reconcile_effect_not_applied( + &self, + effect_id: &str, + actor: &str, + reason: &str, + now: DateTime, + ) -> Result<(), StoreError> { + let identity = format!("{effect_id}\0{actor}\0{reason}\0{}", now.to_rfc3339()); + self.reconcile_effect( + &EffectReconciliationRequest { + reconciliation_id: format!( + "reconciliation-{}", + hex::encode(Sha256::digest(identity.as_bytes())) + ), + effect_id: effect_id.to_owned(), + status: ReconciliationStatus::NotApplied, + actor: actor.to_owned(), + reason: reason.to_owned(), + evidence: serde_json::json!({"source": "legacy-api"}), + result: None, + result_schema: None, + authorization: serde_json::json!({"kind": "explicit-operator-action"}), + compensation_effect_id: None, + trace_id: "operator-reconciliation".to_owned(), + }, + now, + ) + .map(|_| ()) + } + + pub fn latest_effect_reconciliation( + &self, + effect_id: &str, + ) -> Result, StoreError> { + self.connection + .lock() + .query_row( + "SELECT reconciliation_id, effect_id, run_id, format_version, status, actor, reason, evidence_json, result_json, result_schema_json, authorization_json, compensation_effect_id, supersedes_id, trace_id, created_at + FROM effect_reconciliations + WHERE effect_id = ?1 + ORDER BY created_at DESC, rowid DESC + LIMIT 1", + [effect_id], + decode_reconciliation_row, + ) + .optional()? + .map(reconciliation_from_row) + .transpose() + } + + pub fn effect_reconciliations( + &self, + effect_id: &str, + ) -> Result, StoreError> { + let connection = self.connection.lock(); + let mut statement = connection.prepare( + "SELECT reconciliation_id, effect_id, run_id, format_version, status, actor, reason, evidence_json, result_json, result_schema_json, authorization_json, compensation_effect_id, supersedes_id, trace_id, created_at + FROM effect_reconciliations + WHERE effect_id = ?1 + ORDER BY created_at, rowid", + )?; + statement + .query_map([effect_id], decode_reconciliation_row)? + .map(|row| reconciliation_from_row(row?)) + .collect() + } + + pub fn run_effect_reconciliations( + &self, + run_id: &str, + ) -> Result, StoreError> { + let connection = self.connection.lock(); + let mut statement = connection.prepare( + "SELECT reconciliation_id, effect_id, run_id, format_version, status, actor, reason, evidence_json, result_json, result_schema_json, authorization_json, compensation_effect_id, supersedes_id, trace_id, created_at + FROM effect_reconciliations + WHERE run_id = ?1 + ORDER BY created_at, rowid", + )?; + statement + .query_map([run_id], decode_reconciliation_row)? + .map(|row| reconciliation_from_row(row?)) + .collect() } pub fn latest_effect_for_task( @@ -2445,6 +2872,8 @@ impl SqliteStore { "artifact_blobs", "artifact_refs", "artifact_ingests", + "run_upgrades", + "effect_reconciliations", ]; if !allowed.contains(&table) { return Err(StoreError::Incompatible( @@ -2474,6 +2903,8 @@ impl SqliteStore { artifact_blobs: count("artifact_blobs")?, artifact_references: count("artifact_refs")?, artifact_ingests: count("artifact_ingests")?, + run_upgrades: count("run_upgrades")?, + effect_reconciliations: count("effect_reconciliations")?, }) } @@ -2505,6 +2936,8 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { (4_u32, MIGRATION_4), (5_u32, MIGRATION_5), (6_u32, MIGRATION_6), + (7_u32, MIGRATION_7), + (8_u32, MIGRATION_8), ]; for (version, sql) in migrations .into_iter() @@ -2585,6 +3018,103 @@ fn append_audit_tx( Ok(()) } +fn append_trace_tx( + transaction: &Transaction<'_>, + run_id: &str, + trace_id: &str, + event: &Value, + now: DateTime, +) -> Result<(), StoreError> { + let sequence: i64 = transaction.query_row( + "SELECT COALESCE(MAX(sequence), 0) + 1 FROM trace_events WHERE run_id = ?1", + [run_id], + |row| row.get(0), + )?; + transaction.execute( + "INSERT INTO trace_events (run_id, sequence, trace_id, event_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + run_id, + sequence, + trace_id, + encode(event)?, + now.to_rfc3339() + ], + )?; + Ok(()) +} + +type ReconciliationRow = ( + String, + String, + String, + u32, + String, + String, + String, + String, + Option, + Option, + String, + Option, + Option, + String, + String, +); + +fn decode_reconciliation_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + row.get(11)?, + row.get(12)?, + row.get(13)?, + row.get(14)?, + )) +} + +fn reconciliation_from_row( + row: ReconciliationRow, +) -> Result { + if row.3 != 1 { + return Err(StoreError::Incompatible(format!( + "effect reconciliation format version {}", + row.3 + ))); + } + Ok(EffectReconciliationRecord { + reconciliation_id: row.0, + effect_id: row.1, + run_id: row.2, + format_version: row.3, + status: decode_enum(&row.4, "effect_reconciliation.status")?, + actor: row.5, + reason: row.6, + evidence: decode(&row.7, "effect_reconciliation.evidence")?, + result: row + .8 + .map(|value| decode(&value, "effect_reconciliation.result")) + .transpose()?, + result_schema: row + .9 + .map(|value| decode(&value, "effect_reconciliation.result_schema")) + .transpose()?, + authorization: decode(&row.10, "effect_reconciliation.authorization")?, + compensation_effect_id: row.11, + supersedes_id: row.12, + trace_id: row.13, + created_at: parse_time(&row.14, "effect_reconciliation.created_at")?, + }) +} + type ArtifactReferenceRow = ( String, String, @@ -2868,15 +3398,28 @@ spec: .expect("complete task"); } - fn create_version_four_database(path: &Path) { + fn create_version_database(path: &Path, version: u32) { let connection = Connection::open(path).expect("raw connection"); - connection.execute_batch(MIGRATION_1).expect("v1 schema"); - connection.execute_batch(MIGRATION_2).expect("v2 schema"); - connection.execute_batch(MIGRATION_3).expect("v3 schema"); - connection.execute_batch(MIGRATION_4).expect("v4 schema"); + for (index, migration) in [ + MIGRATION_1, + MIGRATION_2, + MIGRATION_3, + MIGRATION_4, + MIGRATION_5, + MIGRATION_6, + MIGRATION_7, + ] + .into_iter() + .enumerate() + .take(usize::try_from(version).expect("version")) + { + connection + .execute_batch(migration) + .unwrap_or_else(|error| panic!("v{} schema: {error}", index + 1)); + } connection - .pragma_update(None, "user_version", 4) - .expect("v4 marker"); + .pragma_update(None, "user_version", version) + .expect("version marker"); } #[test] @@ -3004,6 +3547,275 @@ spec: ); } + #[test] + fn reconciliation_is_immutable_audited_traced_and_rejects_contradictions() { + let store = SqliteStore::open_memory().expect("store"); + create(&store, "run"); + let effect = EffectRequest::new( + "run", + "one", + 1, + 1, + "external.publish", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"record": "x"}), + "publish record", + "trace", + ); + let now = Utc::now(); + store + .record_effect_request(&effect, now) + .expect("record effect"); + store.mark_effect_started(&effect.id, now).expect("start"); + store + .mark_effect_uncertain(&effect.id, "unknown", now) + .expect("uncertain"); + let applied = EffectReconciliationRequest { + reconciliation_id: "reconciliation-applied-1".to_owned(), + effect_id: effect.id.clone(), + status: ReconciliationStatus::Applied, + actor: "operator".to_owned(), + reason: "external record exists".to_owned(), + evidence: serde_json::json!({"externalId": "record-1"}), + result: Some(serde_json::json!({"externalId": "record-1"})), + result_schema: Some(serde_json::json!({"type": "object"})), + authorization: serde_json::json!({"kind": "manual"}), + compensation_effect_id: None, + trace_id: "trace-reconcile".to_owned(), + }; + let first = store + .reconcile_effect(&applied, now) + .expect("applied reconciliation"); + assert_eq!(first.status, ReconciliationStatus::Applied); + assert!( + store + .unresolved_effects("run") + .expect("resolved") + .is_empty() + ); + let source = store.load_effect(&effect.id).expect("immutable source"); + assert_eq!(source.status, EffectStatus::Uncertain); + assert!(!source.confirmed); + + let mut superseding = applied.clone(); + superseding.reconciliation_id = "reconciliation-applied-2".to_owned(); + superseding.reason = "external record independently verified".to_owned(); + let second = store + .reconcile_effect(&superseding, now + chrono::Duration::seconds(1)) + .expect("same-outcome supersession"); + assert_eq!( + second.supersedes_id.as_deref(), + Some("reconciliation-applied-1") + ); + let mut contradiction = superseding; + contradiction.reconciliation_id = "reconciliation-contradiction".to_owned(); + contradiction.status = ReconciliationStatus::NotApplied; + contradiction.result = None; + assert!(matches!( + store.reconcile_effect(&contradiction, now + chrono::Duration::seconds(2)), + Err(StoreError::Incompatible(_)) + )); + let compensation = EffectRequest::new( + "run", + "one", + 1, + 2, + "external.delete", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Idempotent, + serde_json::json!({"externalId": "record-1"}), + "delete record", + "trace", + ); + store + .record_effect_request(&compensation, now) + .expect("compensation"); + store + .mark_effect_started(&compensation.id, now) + .expect("start compensation"); + store + .complete_effect( + &compensation.id, + Ok(&serde_json::json!({"deleted": true})), + now, + ) + .expect("complete compensation"); + let mut compensated = applied; + compensated.reconciliation_id = "reconciliation-compensated".to_owned(); + compensated.status = ReconciliationStatus::Compensated; + compensated.result = None; + compensated.compensation_effect_id = Some(compensation.id); + let compensated = store + .reconcile_effect(&compensated, now + chrono::Duration::seconds(3)) + .expect("applied to compensated"); + assert_eq!( + compensated.supersedes_id.as_deref(), + Some("reconciliation-applied-2") + ); + assert_eq!( + store + .effect_reconciliations(&effect.id) + .expect("history") + .len(), + 3 + ); + assert!( + store + .audit_events("run") + .expect("audit") + .iter() + .any(|event| event.event_type == "effect.reconciled") + ); + assert!( + store + .trace_events("run") + .expect("traces") + .iter() + .any(|event| event.event["name"] == "effect.reconcile") + ); + } + + #[test] + fn reconciliation_compensation_requires_a_confirmed_same_run_effect() { + let store = SqliteStore::open_memory().expect("store"); + create(&store, "run"); + let now = Utc::now(); + let original = EffectRequest::new( + "run", + "one", + 1, + 1, + "external.publish", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::AtMostOnce, + serde_json::json!({"record": "x"}), + "publish record", + "trace", + ); + store + .record_effect_request(&original, now) + .expect("original"); + store + .mark_effect_started(&original.id, now) + .expect("start original"); + store + .complete_effect( + &original.id, + Ok(&serde_json::json!({"externalId": "record-1"})), + now, + ) + .expect("complete original"); + let compensation = EffectRequest::new( + "run", + "one", + 1, + 2, + "external.delete", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Idempotent, + serde_json::json!({"externalId": "record-1"}), + "delete record", + "trace", + ); + store + .record_effect_request(&compensation, now) + .expect("compensation"); + store + .mark_effect_started(&compensation.id, now) + .expect("start compensation"); + let request = EffectReconciliationRequest { + reconciliation_id: "reconciliation-compensated".to_owned(), + effect_id: original.id.clone(), + status: ReconciliationStatus::Compensated, + actor: "operator".to_owned(), + reason: "record was deleted".to_owned(), + evidence: serde_json::json!({"externalId": "record-1", "deleted": true}), + result: None, + result_schema: None, + authorization: serde_json::json!({"kind": "manual"}), + compensation_effect_id: Some(compensation.id.clone()), + trace_id: "trace-reconcile".to_owned(), + }; + assert!(matches!( + store.reconcile_effect(&request, now), + Err(StoreError::Incompatible(_)) + )); + store + .complete_effect( + &compensation.id, + Ok(&serde_json::json!({"deleted": true})), + now, + ) + .expect("complete compensation"); + let record = store.reconcile_effect(&request, now).expect("compensated"); + assert_eq!(record.status, ReconciliationStatus::Compensated); + assert_eq!( + record.compensation_effect_id.as_deref(), + Some(compensation.id.as_str()) + ); + } + + #[test] + fn not_applied_reconciliation_can_only_supersede_the_same_outcome() { + let store = SqliteStore::open_memory().expect("store"); + create(&store, "run"); + let now = Utc::now(); + let effect = EffectRequest::new( + "run", + "one", + 1, + 1, + "external.publish", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"record": "x"}), + "publish", + "trace", + ); + store.record_effect_request(&effect, now).expect("effect"); + store.mark_effect_started(&effect.id, now).expect("start"); + store + .mark_effect_uncertain(&effect.id, "unknown", now) + .expect("uncertain"); + let request = EffectReconciliationRequest { + reconciliation_id: "not-applied-1".to_owned(), + effect_id: effect.id.clone(), + status: ReconciliationStatus::NotApplied, + actor: "operator".to_owned(), + reason: "no remote record".to_owned(), + evidence: serde_json::json!({"query": "empty"}), + result: None, + result_schema: None, + authorization: serde_json::json!({"kind": "manual"}), + compensation_effect_id: None, + trace_id: "trace-reconcile".to_owned(), + }; + store.reconcile_effect(&request, now).expect("not applied"); + let mut superseding = request.clone(); + superseding.reconciliation_id = "not-applied-2".to_owned(); + assert_eq!( + store + .reconcile_effect(&superseding, now + chrono::Duration::seconds(1)) + .expect("supersede") + .supersedes_id + .as_deref(), + Some("not-applied-1") + ); + superseding.reconciliation_id = "contradictory-applied".to_owned(); + superseding.status = ReconciliationStatus::Applied; + superseding.result = Some(serde_json::json!({"record": "x"})); + assert!(matches!( + store.reconcile_effect(&superseding, now + chrono::Duration::seconds(2)), + Err(StoreError::Incompatible(_)) + )); + } + #[test] fn tool_effect_and_call_completion_commit_atomically() { let store = SqliteStore::open_memory().expect("store"); @@ -3108,23 +3920,31 @@ spec: fn upgrades_a_version_one_database_transactionally() { let directory = tempdir().expect("temp dir"); let path = directory.path().join("runtime.db"); - let connection = Connection::open(&path).expect("raw connection"); - connection.execute_batch(MIGRATION_1).expect("v1 schema"); - connection - .pragma_update(None, "user_version", 1) - .expect("v1 marker"); - drop(connection); + create_version_database(&path, 1); let store = SqliteStore::open(&path).expect("upgrade"); assert_eq!(store.schema_version(), DATABASE_SCHEMA_VERSION); assert_eq!(store.stats().expect("stats").long_term_memory, 0); } + #[test] + fn upgrades_every_retained_database_schema_fixture() { + for version in 1..DATABASE_SCHEMA_VERSION { + let directory = tempdir().expect("temp dir"); + let path = directory.path().join(format!("runtime-v{version}.db")); + create_version_database(&path, version); + let store = SqliteStore::open(&path) + .unwrap_or_else(|error| panic!("upgrade schema {version}: {error}")); + assert_eq!(store.schema_version(), DATABASE_SCHEMA_VERSION); + assert_eq!(store.stats().expect("stats").run_upgrades, 0); + } + } + #[test] fn upgrades_the_pre_repair_schema_and_creates_repair_records() { let directory = tempdir().expect("temp dir"); let path = directory.path().join("runtime.db"); - create_version_four_database(&path); + create_version_database(&path, 4); let store = SqliteStore::open(&path).expect("upgrade"); assert_eq!(store.schema_version(), DATABASE_SCHEMA_VERSION); @@ -3164,7 +3984,7 @@ spec: fn interrupted_repair_migration_can_restart_cleanly() { let directory = tempdir().expect("temp dir"); let path = directory.path().join("runtime.db"); - create_version_four_database(&path); + create_version_database(&path, 4); let mut connection = Connection::open(&path).expect("raw connection"); let transaction = connection.transaction().expect("migration transaction"); transaction @@ -3210,6 +4030,83 @@ spec: assert_eq!(store.stats().expect("stats").runs, 0); } + #[test] + fn legacy_run_upgrade_rolls_back_task_mutations_on_failure() { + let directory = tempdir().expect("temp dir"); + let database = directory.path().join("runtime.db"); + let artifact_path = directory.path().join("legacy.txt"); + std::fs::write(&artifact_path, b"legacy artifact").expect("artifact"); + let store = SqliteStore::open(&database).expect("store"); + create(&store, "legacy"); + let now = Utc::now(); + begin_task(&store, "legacy"); + store + .transition_task( + "legacy", + "one", + TaskState::Succeeded, + Some(&serde_json::json!({"ok": true})), + None, + None, + now, + "trace", + ) + .expect("legacy success"); + store + .update_run_state( + "legacy", + RunState::Succeeded, + Some(&serde_json::json!({"ok": true})), + now, + "trace", + ) + .expect("terminal"); + let artifact = store + .ingest_artifact("legacy", "one", &artifact_path, "legacy.txt", 1024, now) + .expect("ingest"); + let metadata = TaskCompletionMetadata { + execution: TaskExecutionMetadata { + metadata_version: 1, + definition_fingerprint: "definition".to_owned(), + input_digest: "input".to_owned(), + output_contract_fingerprint: "contract".to_owned(), + }, + output_digest: "output".to_owned(), + state_delta: serde_json::json!({"set": {}, "remove": []}), + state_delta_digest: "state".to_owned(), + artifact_manifest: vec![artifact], + }; + let result = store.apply_legacy_run_upgrade( + "upgrade", + "legacy", + &serde_json::json!({"dryRun": false}), + &[ + LegacyTaskUpgrade { + task_id: "one".to_owned(), + metadata: metadata.clone(), + provenance: serde_json::json!({"confidence": "proven"}), + }, + LegacyTaskUpgrade { + task_id: "missing".to_owned(), + metadata, + provenance: serde_json::json!({"confidence": "proven"}), + }, + ], + now, + "trace", + ); + + assert!(matches!(result, Err(StoreError::Incompatible(_)))); + assert_eq!( + store.list_tasks("legacy").expect("tasks")[0].metadata_version, + None + ); + let stats = store.stats().expect("stats"); + assert_eq!(stats.run_upgrades, 0); + assert_eq!(stats.artifact_references, 0); + assert_eq!(stats.artifact_ingests, 1); + } + #[test] fn concurrent_readers_and_bounded_lock_wait_succeed() { let directory = tempdir().expect("temp dir"); diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index 35cf5cd..bb3343e 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -12,7 +12,7 @@ SQLite is the local history and correctness boundary. Run, task, effect, approva An effect ID is SHA-256 over run ID, task ID, task attempt, ordinal, operation, and input digest. Each record carries its format version, idempotency key, effect class, risk, status, request/result or error, timestamps, trace correlation, and confirmation flag. The request commits before the executor starts. This supports deterministic reuse of completed results but does not prove exactly-once behavior in an external system. -Pure operations need no external guarantee. Idempotent and keyed effects may be safely retried only when their implementation contract says so. Model calls and unknown remote mutations are treated at-most-once after start: a crash in the acknowledgement window creates an uncertain effect requiring operator reconciliation or an explicit fork. This is deliberately more conservative than silent at-least-once replay. +Pure operations need no external guarantee. Idempotent and keyed effects may be safely retried only when their implementation contract says so. Model calls and unknown remote mutations are treated at-most-once after start: a crash in the acknowledgement window creates an uncertain effect requiring operator reconciliation. Reconciliation appends an immutable `applied`, `not_applied`, or `compensated` conclusion with evidence; it does not rewrite the source effect. This is deliberately more conservative than silent at-least-once replay. Working-memory replacement, the task transition, checkpoint, and audit event commit in one SQLite transaction. Tool-effect and tool-call terminal status also commit together, so inspection cannot observe one as completed while the other remains started. On resume, a confirmed memory-write effect is applied to the reconstructed working-memory value during the succeeding transition. Long-term memory is an external effect and is not rolled back by replay. @@ -20,7 +20,7 @@ Successful workspace mutations are ingested into the local content-addressed art Repair planning is effect-free. A source task is reusable only when its metadata version, definition, dependencies, resolved inputs, output contract/value, state delta, content-addressed artifacts, and effect certainty are compatible. The repair run stores the reused result, artifact references, and provenance in its own rows, so later source-row or workspace deletion does not break it. Missing or corrupt CAS bytes block reuse before a repair run is created. -Cancellation is both an injected token and a durable run flag. CLI SIGINT and SIGTERM cancel in-flight async calls and return exit `130`; `agentctl cancel` records a request for another process to observe. An overall CLI deadline can be set with `--timeout-seconds`, in addition to task/tool/provider/protocol bounds. A provider, tool, process, MCP, or A2A timeout/cancellation/transport loss after dispatch marks the effect `uncertain`; resume refuses to guess and requires reconciliation or an explicit fork. +Cancellation is both an injected token and a durable run flag. CLI SIGINT and SIGTERM cancel in-flight async calls and return exit `130`; `agentctl cancel` records a request for another process to observe. An overall CLI deadline can be set with `--timeout-seconds`, in addition to task/tool/provider/protocol bounds. A provider, tool, process, MCP, or A2A timeout/cancellation/transport loss after dispatch marks the effect `uncertain`; resume refuses to guess and requires reconciliation. An applied reconciliation supplies a validated recorded result. A not-applied or compensated reconciliation resumes with a fresh task and effect attempt. A repaired agent task starts a fresh provider session. Source `previous_response_id`, incomplete turns, pending tool calls, and reasoning state are not copied. Validated task output and reconstructed memory are the only cross-task/cross-run dataflow. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index ec6aef3..62cb649 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -45,9 +45,9 @@ These are useful extensions but are not required by the product thesis. They nee - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. - At-most-once model/remote calls can become uncertain in the dispatch/acknowledgement window. Inspect and reconcile externally; use `fork` only when fresh effects are knowingly acceptable. -- Selective repair requires task metadata version 1. Successful tasks from databases created before schema 5 remain inspectable but must execute from an earlier repair root or a full fork. +- Successful tasks from databases created before schema 5 require explicit `runs analyze`/`runs upgrade`. Only provable metadata is imported; unprovable boundaries are returned as conservative safe repair roots. - Automatic artifact ingestion covers regular files up to 16 MiB reported by successful built-in workspace-mutation results. Larger outputs and artifacts produced only by opaque external effects require an explicit bounded import/export integration. The local CAS must be backed up with SQLite; missing or corrupt blob bytes block repair before run creation and report the expected artifact identity. -- A confirmed non-idempotent mutation in a repair closure remains blocked. The only built-in reconciliation outcome is an operator-confirmed `not-applied` result for a started or uncertain effect; compensation and provider-specific deduplication workflows are not implemented. +- An applied non-idempotent mutation in a repair closure remains blocked from duplicate execution unless a confirmed compensation is linked. Reconciliation supports immutable `applied`, `not_applied`, and `compensated` records, validated results, policy authorization, and operation-specific verification hooks; it does not provide exactly-once delivery. - Retry remains a bounded same-run task policy. There is no separate command that creates a new terminal-source retry run for an unchanged workflow; use repair with an unchanged target definition and explicit roots when its compatibility checks fit. - Tool-using OpenAI/Azure agents require stored-response continuation. `store: false` is rejected until stateless response-item replay is implemented. - Anthropic, Google, Azure OpenAI, MCP, and A2A are native and mock-tested in this release, not live-tested. Only the OpenAI GPT-5.6 tool path has live end-to-end evidence. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 4483fe5..5e86143 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -68,9 +68,9 @@ cargo xtask acceptance-container | Workstream | Focused evidence | Composite evidence | Status | | --- | --- | --- | --- | -| Artifact CAS | pending | pending | open | -| Legacy upgrades | pending | pending | open | -| Reconciliation | pending | pending | open | +| Artifact CAS | 19 store tests and 38 runtime tests | CLI acceptance and hardened OCI acceptance passed | verified | +| Legacy upgrades | all retained schema fixtures, dry-run, rollback, import, boundary, repair/replay tests | migration verification command added; full composite rerun pending | verified | +| Reconciliation | immutable transition matrix, schema/tool/hook/policy, repair and resume tests | full composite rerun pending | verified | | Terminal retry | pending | pending | open | | Parallel/dynamic workflows | pending | pending | open | | Conditions/loops/sub-workflows | pending | pending | open | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index aa1199d..f049696 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -35,8 +35,8 @@ complete, every entry must have exactly one final disposition: | ID | Category | Program state | Intended final disposition | | --- | --- | --- | --- | | ART-001 | Durable artifacts | verified | implemented | -| MIG-001 | Legacy selective repair | open | implemented | -| EFX-001 | Effect reconciliation | open | implemented | +| MIG-001 | Legacy selective repair | verified | implemented | +| EFX-001 | Effect reconciliation | verified | implemented | | RET-001 | Terminal-run retry | open | implemented | | ENC-001 | Sensitive-state encryption | open | implemented | | SEC-001 | Secret providers | open | implemented | @@ -97,9 +97,11 @@ complete, every entry must have exactly one final disposition: ### MIG-001: Legacy run analysis and upgrade -- Current behavior: schema-v5 migration preserves old tasks but cannot safely - reuse tasks that lack repair metadata version 1. -- User impact: operators must choose an unnecessarily broad root or full fork. +- Current behavior: `runs analyze` and `runs upgrade` derive only metadata + proven by retained workflow, plan, effects, outputs, and checksummed + checkpoints. Unprovable suffixes receive conservative safe repair roots. +- User impact: compatible proven predecessors remain reusable without a full + fork. - Security or durability impact: fabricating missing fingerprints or deltas would permit unsafe reuse. - Product decision: implement transactional dry-run analysis and an explicit @@ -108,21 +110,27 @@ complete, every entry must have exactly one final disposition: - Required implementation: `runs upgrade` analysis/apply UX, confidence and provenance records, digest derivation, checkpoint-delta reconstruction, artifact import, and earliest-safe-boundary output. -- Migration impact: every retained schema fixture remains readable; upgrades - are additive and source records remain immutable. +- Migration impact: schema 7 records additive transactional upgrades; every + retained schema fixture remains readable and source execution records remain + unchanged. - Tests: schema fixtures 1 through 5, complete/partial/impossible derivation, failed-upgrade rollback, dry run, corrupt checkpoints, and boundary choice. - Examples: legacy analysis followed by selective retry/repair. - Live evidence: not required; the contract is deterministic. - Documentation: database migration, compatibility, and operator guidance. -- Final disposition: pending implementation evidence. +- Final disposition: implemented and verified by retained-schema migration, + dry-run immutability, artifact-import, failed-upgrade rollback, + impossible-proof boundary, selective repair, workspace deletion, and offline + replay tests. ### EFX-001: Complete operator reconciliation -- Current behavior: only a started or uncertain effect can be changed to a - failed `not_applied` state. -- User impact: applied, compensated, or externally completed work cannot be - represented safely. +- Current behavior: source effects are immutable. Versioned reconciliation + records represent `applied`, `not_applied`, and `compensated` conclusions + with effective runtime projection. +- User impact: an operator can resume from a validated applied result, begin a + fresh attempt after not-applied/compensated evidence, and safely unblock a + compatible repair. - Security or durability impact: operators may resort to unsafe forks or out-of-band database edits. - Product decision: preserve immutable source effects and append versioned @@ -131,15 +139,18 @@ complete, every entry must have exactly one final disposition: `not_applied`, and `compensated`; identity, timestamp, reason, evidence, optional validated result, supersession rules, compensation linkage, audit, trace, policy, and non-interactive behavior. -- Migration impact: new reconciliation table and effective-effect projection; - existing `not_applied` audits migrate to records when provable. +- Migration impact: schema 8 adds reconciliation history and effective-effect + projection. Existing source effects are never rewritten. - Tests: every transition, contradictory decisions, supersession, wrong schemas, operator policy, repair/resume/retry integration, idempotency keys, and transaction rollback. - Examples: operational workflow with manual applied and compensated outcomes. - Live evidence: selective repair after an explicitly reconciled mock effect. - Documentation: effect recovery and honest external-state semantics. -- Final disposition: pending implementation evidence. +- Final disposition: implemented and verified by transition/supersession, + contradiction, compensation-link, immutable-source, audit/trace, + result-schema/tool-contract/hook, policy approval, repair, and both resume + paths. ### RET-001: Terminal-run retry diff --git a/docs/generated/CLI.md b/docs/generated/CLI.md index 3c17cfd..1f78bea 100644 --- a/docs/generated/CLI.md +++ b/docs/generated/CLI.md @@ -17,6 +17,7 @@ Commands: replay Reconstruct a terminal run only from recorded state and results fork Create a new run from a prior workflow with fresh effects repair Create a new run that reuses compatible upstream results and executes a repaired suffix + runs Analyze or upgrade retained legacy run records for selective reuse cancel Durably request cancellation inspect Inspect durable run, task, and audit state effects Inspect or narrowly reconcile uncertain effects @@ -191,6 +192,60 @@ Options: -h, --help Print help ``` +## `agentctl runs` + +```text +Analyze or upgrade retained legacy run records for selective reuse + +Usage: agentctl runs [OPTIONS] + +Commands: + analyze Prove reusable legacy metadata without changing the source run + upgrade Transactionally persist every legacy field that can be proven + +Options: + --db [default: .agentctl/runtime.db] + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl runs analyze` + +```text +Prove reusable legacy metadata without changing the source run + +Usage: agentctl runs analyze [OPTIONS] + +Arguments: + + +Options: + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl runs upgrade` + +```text +Transactionally persist every legacy field that can be proven + +Usage: agentctl runs upgrade [OPTIONS] + +Arguments: + + +Options: + --dry-run + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + ## `agentctl cancel` ```text @@ -235,6 +290,7 @@ Inspect or narrowly reconcile uncertain effects Usage: agentctl effects [OPTIONS] Commands: + list inspect reconcile @@ -246,10 +302,10 @@ Options: -h, --help Print help ``` -## `agentctl effects inspect` +## `agentctl effects list` ```text -Usage: agentctl effects inspect [OPTIONS] +Usage: agentctl effects list [OPTIONS] Arguments: @@ -262,22 +318,54 @@ Options: -h, --help Print help ``` +## `agentctl effects inspect` + +```text +Usage: agentctl effects inspect [OPTIONS] + +Arguments: + + +Options: + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + ## `agentctl effects reconcile` ```text -Usage: agentctl effects reconcile [OPTIONS] --outcome --reason +Usage: agentctl effects reconcile [OPTIONS] --status --reason Arguments: Options: - --outcome [possible values: not-applied] - --output [default: human] [possible values: human, json] - --actor [default: cli-user] - --color [default: auto] [possible values: auto, always, never] + --output + [default: human] [possible values: human, json] + --status + [possible values: applied, not-applied, compensated] + --actor + [default: cli-user] + --color + [default: auto] [possible values: auto, always, never] --reason + --verbose - -h, --help Print help + + --evidence-file + + --result-file + + --result-schema-file + + --compensation-effect + + --approved + + -h, --help + Print help ``` ## `agentctl approvals` @@ -464,6 +552,104 @@ Options: -h, --help Print help ``` +## `agentctl artifacts` + +```text +Inspect, verify, export, or collect durable artifacts + +Usage: agentctl artifacts [OPTIONS] + +Commands: + list + inspect + verify + export + gc + +Options: + --db [default: .agentctl/runtime.db] + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl artifacts list` + +```text +Usage: agentctl artifacts list [OPTIONS] + +Options: + --output [default: human] [possible values: human, json] + --run + --color [default: auto] [possible values: auto, always, never] + --task + --verbose + -h, --help Print help +``` + +## `agentctl artifacts inspect` + +```text +Usage: agentctl artifacts inspect [OPTIONS] + +Arguments: + + +Options: + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl artifacts verify` + +```text +Usage: agentctl artifacts verify [OPTIONS] [DIGEST] + +Arguments: + [DIGEST] + +Options: + --all + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl artifacts export` + +```text +Usage: agentctl artifacts export [OPTIONS] + +Arguments: + + + +Options: + --output [default: human] [possible values: human, json] + --overwrite + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl artifacts gc` + +```text +Usage: agentctl artifacts gc [OPTIONS] + +Options: + --older-than-days [default: 30] + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --dry-run + --verbose + -h, --help Print help +``` + ## `agentctl db` ```text diff --git a/docs/guides/EFFECT_RECONCILIATION.md b/docs/guides/EFFECT_RECONCILIATION.md new file mode 100644 index 0000000..f2927ac --- /dev/null +++ b/docs/guides/EFFECT_RECONCILIATION.md @@ -0,0 +1,60 @@ +# Reconcile an uncertain effect + +An effect can be externally applied even when the process loses its acknowledgement. `agentctl` records that ambiguity as `started` or `uncertain` and refuses to guess. Reconciliation appends an operator conclusion; it never rewrites the source effect. + +List a run and inspect one effect: + +```text +agentctl effects --db .agentctl/runtime.db list RUN_ID --output json +agentctl effects --db .agentctl/runtime.db inspect EFFECT_ID --output json +``` + +Verify the external system using an independent identifier or query. Then record exactly one of these conclusions. + +Not applied, so a fresh task attempt may dispatch it: + +```text +agentctl effects --db .agentctl/runtime.db reconcile EFFECT_ID \ + --status not-applied \ + --actor operator-name \ + --reason "remote lookup returned no record" \ + --evidence-file evidence.json +``` + +Applied, with the externally confirmed result needed to resume: + +```text +agentctl effects --db .agentctl/runtime.db reconcile EFFECT_ID \ + --status applied \ + --actor operator-name \ + --reason "remote lookup confirmed record ext-123" \ + --evidence-file evidence.json \ + --result-file result.json \ + --result-schema-file result.schema.json +``` + +Compensated, linked to a confirmed compensation effect: + +```text +agentctl effects --db .agentctl/runtime.db reconcile EFFECT_ID \ + --status compensated \ + --actor operator-name \ + --reason "the created record was deleted" \ + --evidence-file evidence.json \ + --compensation-effect COMPENSATION_EFFECT_ID +``` + +When policy requires approval, add `--approved`. This is explicit non-interactive authorization and is stored with the operator identity and policy decision. It cannot override a policy denial. + +Applied results are validated against a supplied JSON Schema. Model results must also decode as a provider response. Registered tools validate results against their output contracts, and extensions can register an operation-specific reconciliation hook. + +The transition rules prevent contradictory history: + +- `applied` may be superseded by another `applied` record or progress to `compensated`. +- `not_applied` may only be superseded by another `not_applied` record. +- `compensated` may only be superseded by another `compensated` record. +- compensation requires a different, same-run effect that is confirmed applied. + +Every record includes actor, timestamp, reason, evidence, optional result and schema, authorization, trace, supersession, and compensation linkage. Audit and trace records are written in the same transaction. + +For a non-terminal interrupted run, `resume` consumes an `applied` result without redispatch. A `not_applied` or `compensated` conclusion starts a fresh task attempt with a new effect identity. For terminal sources, create a compatible repair run. Unresolved non-idempotent effects are never silently repeated. diff --git a/docs/guides/LEGACY_RUN_UPGRADE.md b/docs/guides/LEGACY_RUN_UPGRADE.md new file mode 100644 index 0000000..9c967ce --- /dev/null +++ b/docs/guides/LEGACY_RUN_UPGRADE.md @@ -0,0 +1,39 @@ +# Upgrade a legacy run for selective repair + +Runs written before database schema 5 can contain successful outputs without the fingerprints, state deltas, and artifact identities required for safe reuse. `agentctl` does not invent those fields. It analyzes retained workflow, plan, effect, output, and checksummed checkpoint records and reports which task boundaries are provable. + +First migrate the database and run a read-only analysis: + +```text +agentctl db --db .agentctl/runtime.db migrate +agentctl runs --db .agentctl/runtime.db analyze RUN_ID --output json +``` + +`upgradeableTasks` lists successful tasks whose complete metadata can be derived. Each task includes field-level provenance. `unavailableTasks` lists boundaries with missing or contradictory proof. `recommendedRepairRoots` is the conservative earliest safe suffix: every later task is covered unless dependency closure already covers it. + +`analyze` and `upgrade --dry-run` do not write: + +```text +agentctl runs --db .agentctl/runtime.db upgrade RUN_ID --dry-run --output json +``` + +Apply the proven subset explicitly: + +```text +agentctl runs --db .agentctl/runtime.db upgrade RUN_ID --output json +``` + +The upgrade imports provable regular-file artifacts into the sibling content-addressed store, verifies their recorded identity, updates legacy task metadata, appends an upgrade record and audit event, and checkpoints the run in one SQLite transaction. It preserves the source output, effect, checkpoint, and workflow records. A failed task update rolls back all metadata and references from that upgrade. Artifact ingestion leases may remain temporarily and are reclaimed by normal artifact GC. + +Use the returned roots with the corrected workflow: + +```text +agentctl repair repaired.workflow.yaml RUN_ID \ + --from SAFE_ROOT \ + --plan \ + --db .agentctl/runtime.db +``` + +If several roots are returned, pass `--from` once for each root. A task that remains unprovable is executed again; compatible proven predecessors remain reusable. After a successful upgrade and repair, content-addressed artifacts remain usable if the original workspace file is deleted. + +Back up SQLite, its WAL state, and the sibling `artifacts/` directory together before migration. `cargo xtask migration-verify` exercises every retained database schema fixture. diff --git a/docs/guides/repair-a-failed-workflow.md b/docs/guides/repair-a-failed-workflow.md index c0bfc1d..65dc6e2 100644 --- a/docs/guides/repair-a-failed-workflow.md +++ b/docs/guides/repair-a-failed-workflow.md @@ -137,22 +137,23 @@ flowchart TD N -->|no or unknown| H["Choose a safe business remediation
or broader fresh execution"] ``` -Inspect effects for the failed boundary: +List effects for the failed boundary, then inspect the selected effect: ```bash -agentctl effects --db .agentctl/runtime.db inspect SOURCE_RUN_ID --task publish +agentctl effects --db .agentctl/runtime.db list SOURCE_RUN_ID --task publish +agentctl effects --db .agentctl/runtime.db inspect EFFECT_ID ``` If an effect is `started` or `uncertain` and an operator has verified that it did not happen: ```bash agentctl effects --db .agentctl/runtime.db reconcile EFFECT_ID \ - --outcome not-applied \ + --status not-applied \ --reason "remote system confirms no record" \ --actor operator-name ``` -There is no generic force option and no exactly-once claim. Confirmed non-idempotent effects stay blocked because repeating them may duplicate external work. Normal policy, approval, timeout, retry, and cancellation behavior applies to every fresh task. +Use `--status applied --result-file result.json` when the effect happened and resume needs its externally confirmed result. Use `--status compensated --compensation-effect EFFECT_ID` only after a distinct compensation effect is confirmed. There is no generic force option and no exactly-once claim. An applied non-idempotent effect stays blocked from duplicate fresh execution until it has a valid compensation record. Normal policy, approval, timeout, retry, and cancellation behavior applies to every fresh task. See [Effect reconciliation](EFFECT_RECONCILIATION.md). A repaired agent begins a new provider session. It receives target instructions and tools plus validated upstream output and reconstructed memory. It never receives the failed source task's `previous_response_id`, incomplete turn, pending call, or reasoning state. Within the new repaired task, normal multi-turn continuation still applies. @@ -184,7 +185,7 @@ Recorded replay has a new replay run ID but the same semantic outputs. It dispat | `state_delta_missing` or `state_delta_invalid` | Successful boundary-state metadata is absent or corrupt. | Select the task as an earlier root; do not edit the database. | | `artifact_integrity` | A content-addressed artifact is missing or corrupt. The block reports its logical path, expected digest, and expected size. | Restore the database and sibling CAS from a consistent backup, or select its producer as an earlier repair root. | | `unresolved_reused_effect` | A nominally successful reusable task retains a started or uncertain effect. | Reconcile external reality before reuse. | -| `legacy_task_metadata` | The source predates repair metadata v1. | Use an earlier root or a full fork. | +| `legacy_task_metadata` | The source predates repair metadata v1. | Run `agentctl runs analyze` and `runs upgrade`, then use the reported safe root. | | `new_task_outside_repair_closure` | A new unrelated task has no result. | Add it as a root or choose an earlier common boundary. | | `unreconciled_effect` | Fresh execution may duplicate a mutation. | Inspect and reconcile external reality first. | diff --git a/docs/reference/DATABASE.md b/docs/reference/DATABASE.md index 0ffb5b1..0d594b3 100644 --- a/docs/reference/DATABASE.md +++ b/docs/reference/DATABASE.md @@ -1,22 +1,24 @@ # Runtime database and migrations -The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `6`. +The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `8`. ## Stored records - runs, source workflow, compiled plan, inputs, output, mode, state, parent linkage, and repair source/root metadata - task states, attempts, output, errors, disposition, source attempt, versioned fingerprints/digests, state delta, artifact manifest, and reuse decision - effects, request/result/error, confirmation, and uncertainty +- immutable effect reconciliation history, operator authorization, evidence, validated results, supersession, and compensation linkage - approvals and resolutions - checksummed checkpoints - ordered audit and trace events - provider sessions and tool calls - namespaced long-term memory with optional expiry - content-addressed blob metadata, logical run/task references, provenance, verification time, and bounded ingestion leases +- legacy-run upgrade analysis and the exact task metadata applied by each upgrade Working memory is stored on the run and in checkpoints. Provider credentials are not stored. Other confidential content may be stored, including prompts, tool output, and remote artifacts. -Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. A repair transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete a repair run. +Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. A repair transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete a repair run. Artifact manifests contain logical path/name, media type, byte size, SHA-256 digest, and CAS-relative path. Blob bytes live under `/artifacts/sha256/`; identical content is stored once. A completed repair/replay receives its own references, so source-row and workspace deletion do not break it. @@ -27,6 +29,10 @@ The store reads SQLite `user_version` and applies forward migrations in order in ```text agentctl db stats --db .agentctl/runtime.db --output json --color never agentctl db migrate --db .agentctl/runtime.db --output json --color never +agentctl runs --db .agentctl/runtime.db analyze RUN_ID --output json +agentctl runs --db .agentctl/runtime.db upgrade RUN_ID --dry-run --output json +agentctl runs --db .agentctl/runtime.db upgrade RUN_ID --output json +agentctl effects --db .agentctl/runtime.db list RUN_ID --output json agentctl artifacts --db .agentctl/runtime.db list --run RUN_ID --output json agentctl artifacts --db .agentctl/runtime.db verify --all --output json agentctl artifacts --db .agentctl/runtime.db export SHA256_DIGEST ./report.bin diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index c3d68c5..4aa5dfd 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -10,6 +10,7 @@ publish = false [dependencies] anyhow.workspace = true hex.workspace = true +rusqlite.workspace = true serde_json.workspace = true sha2.workspace = true tempfile.workspace = true diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 9997150..4642964 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -15,7 +15,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 28; +const ACCEPTANCE_SCENARIOS: usize = 29; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -914,6 +914,72 @@ pub fn run(root: &Path) -> Result<()> { ); uncertain_repair_acceptance(&binary, &workspace, directory.path())?; + scenario( + 29, + "legacy dry-run analysis and transactional upgrade expose a safe repair root", + ); + let connection = rusqlite::Connection::open(&repair_db)?; + connection.execute( + "UPDATE task_states SET metadata_version = NULL, definition_fingerprint = NULL, input_digest = NULL, output_contract_fingerprint = NULL, output_digest = NULL, state_delta_json = NULL, state_delta_digest = NULL, artifact_manifest_json = NULL, reuse_decision_json = NULL WHERE run_id = ?1 AND task_id = 'first'", + [source_run_id], + )?; + drop(connection); + let legacy_analysis = successful_json( + &binary, + &workspace, + &strings([ + "runs", + "--db", + path(&repair_db)?, + "upgrade", + source_run_id, + "--dry-run", + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&legacy_analysis, "/data/upgradeableTasks/0", "first")?; + ensure_eq(&legacy_analysis, "/data/recommendedRepairRoots/0", "second")?; + let still_legacy = successful_json( + &binary, + &workspace, + &strings([ + "runs", + "--db", + path(&repair_db)?, + "analyze", + source_run_id, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&still_legacy, "/data/upgradeableTasks/0", "first")?; + let upgraded = successful_json( + &binary, + &workspace, + &strings([ + "runs", + "--db", + path(&repair_db)?, + "upgrade", + source_run_id, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&upgraded, "/data/upgradedTasks/0", "first")?; + ensure_eq( + &upgraded, + "/data/analysisAfter/recommendedRepairRoots/0", + "second", + )?; + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } @@ -1637,7 +1703,7 @@ fn uncertain_repair_acceptance(binary: &Path, workspace: &Path, directory: &Path "effects", "--db", path(&db)?, - "inspect", + "list", run_id, "--task", "work", @@ -1678,7 +1744,7 @@ fn uncertain_repair_acceptance(binary: &Path, workspace: &Path, directory: &Path path(&db)?, "reconcile", effect_id, - "--outcome", + "--status", "not-applied", "--actor", "acceptance", diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2545c81..c4476a6 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -24,6 +24,7 @@ fn main() -> Result<()> { match command.as_str() { "verify" => verify(&root), "docs-verify" => docs_verify(&root), + "migration-verify" => migration_verify(&root), "acceptance" => acceptance::run(&root), "acceptance-container" => acceptance::container(&root), "acceptance-live-openai" => acceptance::live_openai(&root), @@ -37,7 +38,7 @@ fn main() -> Result<()> { } "help" | "--help" | "-h" => { println!( - "cargo xtask verify\ncargo xtask docs-verify\ncargo xtask acceptance\ncargo xtask acceptance-container\ncargo xtask acceptance-live-openai\ncargo xtask examples-verify\ncargo xtask examples-verify-live-openai\ncargo xtask generate\ncargo xtask package\ncargo xtask secret-scan" + "cargo xtask verify\ncargo xtask docs-verify\ncargo xtask migration-verify\ncargo xtask acceptance\ncargo xtask acceptance-container\ncargo xtask acceptance-live-openai\ncargo xtask examples-verify\ncargo xtask examples-verify-live-openai\ncargo xtask generate\ncargo xtask package\ncargo xtask secret-scan" ); Ok(()) } @@ -45,6 +46,20 @@ fn main() -> Result<()> { } } +fn migration_verify(root: &Path) -> Result<()> { + run( + root, + "cargo", + &[ + "test", + "-p", + "agentctl-store", + "upgrades_every_retained_database_schema_fixture", + "--locked", + ], + ) +} + fn docs_verify(root: &Path) -> Result<()> { println!("[1/6] build documentation test binary"); run(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -290,9 +305,13 @@ fn generated_cli_reference(binary: &Path) -> Result { &["replay"], &["fork"], &["repair"], + &["runs"], + &["runs", "analyze"], + &["runs", "upgrade"], &["cancel"], &["inspect"], &["effects"], + &["effects", "list"], &["effects", "inspect"], &["effects", "reconcile"], &["approvals"], @@ -306,6 +325,12 @@ fn generated_cli_reference(binary: &Path) -> Result { &["schema"], &["migrate"], &["packs"], + &["artifacts"], + &["artifacts", "list"], + &["artifacts", "inspect"], + &["artifacts", "verify"], + &["artifacts", "export"], + &["artifacts", "gc"], &["db"], &["memory"], &["gc"], @@ -661,6 +686,8 @@ fn verify_public_documentation(root: &Path) -> Result<()> { "docs/guides/WORKFLOW_AUTHORING.md", "docs/guides/LOCAL_OPERATION.md", "docs/guides/repair-a-failed-workflow.md", + "docs/guides/LEGACY_RUN_UPGRADE.md", + "docs/guides/EFFECT_RECONCILIATION.md", "docs/guides/CI_CD.md", "docs/guides/TROUBLESHOOTING.md", "docs/reference/YAML.md", From 4419746200679788741524da4ad79a8e6b376ccb Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 16:08:42 +0530 Subject: [PATCH 08/44] feat: add terminal workflow retry --- README.md | 8 +- crates/agentctl-cli/src/main.rs | 133 ++++- crates/agentctl-runtime/src/lib.rs | 529 +++++++++++++++++++- crates/agentctl-store/src/lib.rs | 225 ++++++++- docs/DURABLE_EXECUTION.md | 5 +- docs/LIMITATIONS.md | 2 +- docs/execution/COMPLETENESS_VERIFICATION.md | 2 +- docs/execution/LIMITATION_BURNDOWN.md | 12 +- docs/generated/CLI.md | 29 ++ docs/guides/TERMINAL_RETRY.md | 84 ++++ docs/guides/repair-a-failed-workflow.md | 2 +- docs/reference/DATABASE.md | 6 +- xtask/src/acceptance.rs | 125 ++++- xtask/src/main.rs | 2 + 14 files changed, 1142 insertions(+), 22 deletions(-) create mode 100644 docs/guides/TERMINAL_RETRY.md diff --git a/README.md b/README.md index f962f54..2e7e5d4 100644 --- a/README.md +++ b/README.md @@ -43,16 +43,18 @@ spec: message: hello from agentctl ``` -Use `check` for strict syntax, references, templates, policy, and provider-capability validation. Use `plan` for deterministic order and predictability, `run --check --diff` for a non-mutating preview, `resume` after interruption, `replay` to reconstruct recorded results without effects, `repair` to reuse compatible successful task boundaries with a corrected workflow, and `fork` when a broader fresh execution is intentional. +Use `check` for strict syntax, references, templates, policy, and provider-capability validation. Use `plan` for deterministic order and predictability, `run --check --diff` for a non-mutating preview, `resume` after interruption, `replay` to reconstruct recorded results without effects, `retry` to rerun failed boundaries of an identical terminal workflow, `repair` to reuse compatible successful task boundaries with a corrected workflow, and `fork` when a broader fresh execution is intentional. -Selective repair is planned before execution: +Terminal retry and selective repair are planned before execution: ```text +agentctl retry workflow.yaml SOURCE_RUN_ID --failed --plan +agentctl retry workflow.yaml SOURCE_RUN_ID --failed agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task --plan agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task ``` -See [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. +See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md) and [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). ## Safety boundary diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index 9484713..3b0e572 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -89,6 +89,8 @@ enum Command { Fork(ForkArgs), /// Create a new run that reuses compatible upstream results and executes a repaired suffix. Repair(RepairArgs), + /// Retry failed or selected boundaries of an identical terminal workflow. + Retry(RetryArgs), /// Analyze or upgrade retained legacy run records for selective reuse. Runs(RunsArgs), /// Durably request cancellation. @@ -214,6 +216,36 @@ struct RepairArgs { timeout_seconds: Option, } +#[derive(Debug, Args)] +struct RetryArgs { + file: PathBuf, + source_run_id: String, + #[arg(long, conflicts_with = "from", required_unless_present = "from")] + failed: bool, + #[arg( + long = "from", + conflicts_with = "failed", + required_unless_present = "failed" + )] + from: Vec, + #[arg(long)] + plan: bool, + #[arg(long)] + restart_successful: bool, + #[arg(long)] + reason: Option, + #[arg(long, default_value = ".agentctl/runtime.db")] + db: PathBuf, + #[arg(long)] + interactive: bool, + #[arg(long)] + diff: bool, + #[arg(long)] + workspace: Option, + #[arg(long)] + timeout_seconds: Option, +} + #[derive(Debug, Args)] struct RunsArgs { #[arg(long, default_value = ".agentctl/runtime.db")] @@ -656,6 +688,7 @@ async fn execute(cli: Cli) -> Result { print_outcome(output, &outcome) } Command::Repair(args) => repair_workflow(output, args).await, + Command::Retry(args) => retry_workflow(output, args).await, Command::Runs(args) => runs_command(output, args), Command::Cancel(args) => { let store = open_store(&args.db)?; @@ -717,7 +750,7 @@ async fn execute(cli: Cli) -> Result { audit.len(), traces.len(), ); - let human = if run.mode == RunMode::Repair { + let human = if matches!(run.mode, RunMode::Repair | RunMode::Retry) { let reused = tasks .iter() .filter(|task| task.disposition == TaskDisposition::Reused) @@ -973,6 +1006,100 @@ async fn repair_workflow(output: OutputFormat, args: RepairArgs) -> Result Result { + if !args.plan { + validate_interactive(args.interactive)?; + } + let (workflow, compiled, diagnostics) = load_and_compile(&args.file)?; + let default_base = args + .file + .parent() + .filter(|path| !path.as_os_str().is_empty()); + let base = resolve_base_path(args.workspace.as_deref().or(default_base))?; + let store = open_store(&args.db)?; + let planner = Runtime::new(store.clone(), &base); + let plan = planner + .plan_retry( + &args.source_run_id, + &workflow, + &compiled, + &args.from, + args.failed, + args.restart_successful, + ) + .map_err(map_runtime_error)?; + if args.plan || !plan.compatible { + let human = format!( + "retry plan: {}\nsource: {}\nselection: {}\nreuse: {}\nexecute: {}\nblocked: {}", + if plan.compatible { + "compatible" + } else { + "blocked" + }, + plan.source_run_id, + if plan.failed_only { + "failed tasks".to_owned() + } else { + plan.retry_roots.join(", ") + }, + plan.reused_tasks.join(", "), + plan.rerun_tasks.join(", "), + plan.blocked_reuse + .iter() + .map(|block| format!("{}: {}", block.task_id, block.message)) + .collect::>() + .join("; "), + ); + print_value(output, "RetryPlan", &plan, diagnostics, human)?; + return Ok(if plan.compatible { + EXIT_OK + } else { + EXIT_POLICY + }); + } + let registry = build_registry(&workflow, &base)?; + let runtime = Runtime::new(store, &base).with_registry(registry); + let cancellation = cancellation_token(args.timeout_seconds); + let outcome = runtime + .retry( + &workflow, + &compiled, + plan, + args.reason.as_deref(), + RunOptions { + check: false, + diff: args.diff, + interactive: args.interactive, + }, + &cancellation, + ) + .await + .map_err(map_runtime_error)?; + print_value( + output, + "RetryOutcome", + &outcome, + diagnostics, + format!( + "{} {:?} source={} roots={} reused={} executed={} artifacts={} trace={}", + outcome.run_id, + outcome.state, + outcome.source_run_id, + outcome.retry_roots.join(","), + outcome.reused_tasks.join(","), + outcome.executed_tasks.join(","), + outcome + .artifacts + .iter() + .map(|artifact| artifact.path.as_str()) + .collect::>() + .join(","), + outcome.trace_id, + ), + )?; + Ok(outcome_exit_code(outcome.state)) +} + fn print_outcome( output: OutputFormat, outcome: &agentctl_runtime::RunOutcome, @@ -2197,7 +2324,8 @@ fn map_runtime_error(error: agentctl_runtime::RuntimeError) -> CliError { agentctl_runtime::RuntimeError::Provider(_) => EXIT_REMOTE, agentctl_runtime::RuntimeError::Cancelled => EXIT_CANCELLED, agentctl_runtime::RuntimeError::UncertainEffect { .. } => EXIT_POLICY, - agentctl_runtime::RuntimeError::RepairBlocked { .. } => EXIT_POLICY, + agentctl_runtime::RuntimeError::RepairBlocked { .. } + | agentctl_runtime::RuntimeError::RetryBlocked { .. } => EXIT_POLICY, _ => EXIT_RUN_FAILED, }; CliError { @@ -2230,6 +2358,7 @@ mod tests { "replay", "fork", "repair", + "retry", "runs", "cancel", "inspect", diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index fd1e22a..91030ce 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -332,6 +332,45 @@ pub struct RepairOutcome { pub output: Option, } +pub const RETRY_PLAN_VERSION: &str = "agentctl.dev/retry-plan/v1"; + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RetryPlan { + pub api_version: String, + pub compatible: bool, + pub source_run_id: String, + pub workflow_digest: String, + pub failed_only: bool, + pub retry_roots: Vec, + pub restart_successful: bool, + pub reused_tasks: Vec, + pub rerun_tasks: Vec, + pub blocked_reuse: Vec, + pub fresh_effect_summary: FreshEffectSummary, + pub approval_summary: Vec, + pub estimated_provider_tasks: usize, + pub warnings: Vec, + pub tasks: Vec, + #[serde(skip_serializing)] + repair_plan: RepairPlan, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RetryOutcome { + pub run_id: String, + pub source_run_id: String, + pub trace_id: String, + pub state: RunState, + pub failed_only: bool, + pub retry_roots: Vec, + pub reused_tasks: Vec, + pub executed_tasks: Vec, + pub artifacts: Vec, + pub output: Option, +} + pub const LEGACY_UPGRADE_ANALYSIS_VERSION: &str = "agentctl.dev/legacy-upgrade/v1"; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -427,6 +466,8 @@ pub enum RuntimeError { Json(#[from] serde_json::Error), #[error("repair from run `{source_run_id}` is blocked by {count} compatibility rule(s)")] RepairBlocked { source_run_id: String, count: usize }, + #[error("retry from run `{source_run_id}` is blocked by {count} compatibility rule(s)")] + RetryBlocked { source_run_id: String, count: usize }, } pub struct Runtime { @@ -1944,6 +1985,227 @@ impl Runtime { }) } + pub fn plan_retry( + &self, + source_run_id: &str, + workflow: &Workflow, + compiled: &CompiledPlan, + selected_roots: &[String], + failed_only: bool, + restart_successful: bool, + ) -> Result { + if failed_only && !selected_roots.is_empty() { + return Err(RuntimeError::InvalidState( + "retry accepts either --failed or --from, not both".to_owned(), + )); + } + if !failed_only && selected_roots.is_empty() { + return Err(RuntimeError::InvalidState( + "retry requires --failed or at least one --from task".to_owned(), + )); + } + let source = self.store.load_run(source_run_id)?; + if !source.state.is_terminal() { + return Err(RuntimeError::InvalidState(format!( + "retry source run `{source_run_id}` is not terminal ({:?})", + source.state + ))); + } + let roots = if failed_only { + let roots = self + .store + .list_tasks(source_run_id)? + .into_iter() + .filter(|task| task.state == TaskState::Failed) + .map(|task| task.task_id) + .collect::>(); + if roots.is_empty() { + return Err(RuntimeError::InvalidState(format!( + "source run `{source_run_id}` has no failed tasks" + ))); + } + roots + } else { + selected_roots.to_vec() + }; + let mut repair_plan = self.plan_repair( + source_run_id, + workflow, + compiled, + &roots, + restart_successful, + )?; + if source.workflow_digest != compiled.workflow_digest { + repair_plan.blocked_reuse.push(repair_block( + "$workflow", + "retry_workflow_definition_mismatch", + format!( + "retry requires workflow digest `{}`, found `{}`; use repair for a changed workflow", + source.workflow_digest, compiled.workflow_digest + ), + Some(source.workflow_digest.clone()), + Some(compiled.workflow_digest.clone()), + Vec::new(), + false, + )); + repair_plan.compatible = false; + } + let warnings = vec![ + "retry requires the identical workflow definition and preserves the terminal source" + .to_owned(), + "retry roots and descendants start fresh task and effect attempts".to_owned(), + "compatible successful tasks are materialized without dispatch".to_owned(), + ]; + Ok(RetryPlan { + api_version: RETRY_PLAN_VERSION.to_owned(), + compatible: repair_plan.compatible, + source_run_id: repair_plan.source_run_id.clone(), + workflow_digest: compiled.workflow_digest.clone(), + failed_only, + retry_roots: repair_plan.repair_roots.clone(), + restart_successful, + reused_tasks: repair_plan.reused_tasks.clone(), + rerun_tasks: repair_plan.rerun_tasks.clone(), + blocked_reuse: repair_plan.blocked_reuse.clone(), + fresh_effect_summary: repair_plan.fresh_effect_summary.clone(), + approval_summary: repair_plan.approval_summary.clone(), + estimated_provider_tasks: repair_plan.estimated_provider_tasks, + warnings, + tasks: repair_plan.tasks.clone(), + repair_plan, + }) + } + + #[allow(clippy::too_many_arguments)] + pub async fn retry( + &self, + workflow: &Workflow, + compiled: &CompiledPlan, + plan: RetryPlan, + reason: Option<&str>, + options: RunOptions, + cancellation: &CancellationToken, + ) -> Result { + if !plan.compatible { + return Err(RuntimeError::RetryBlocked { + source_run_id: plan.source_run_id, + count: plan.blocked_reuse.len(), + }); + } + if compiled.workflow_digest != plan.workflow_digest { + return Err(RuntimeError::InvalidState( + "workflow changed after retry planning; create a new retry plan".to_owned(), + )); + } + let selected_roots = if plan.failed_only { + Vec::new() + } else { + plan.retry_roots.clone() + }; + let plan = self.plan_retry( + &plan.source_run_id, + workflow, + compiled, + &selected_roots, + plan.failed_only, + plan.restart_successful, + )?; + if !plan.compatible { + return Err(RuntimeError::RetryBlocked { + source_run_id: plan.source_run_id, + count: plan.blocked_reuse.len(), + }); + } + let source = self.store.load_run(&plan.source_run_id)?; + let run_id = self.ids.next_id("retry"); + let trace_id = self.ids.next_id("trace"); + self.store.create_retry_run( + &run_id, + &plan.source_run_id, + &source.workflow_digest, + API_VERSION, + &serde_json::to_value(workflow)?, + compiled, + &source.inputs, + &plan.repair_plan.reconstructed_memory, + &plan.retry_roots, + plan.failed_only, + reason, + &plan.repair_plan.materialized_tasks, + &serde_json::to_value(&plan.tasks)?, + &self.base_path, + self.clock.now(), + &trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Run, + TracePhase::Started, + "run.retry", + &trace_id, + &run_id, + self.clock.now(), + ) + .attributes( + serde_json::json!({ + "sourceRunId": plan.source_run_id, + "failedOnly": plan.failed_only, + "retryRoots": plan.retry_roots, + "reusedTasks": plan.reused_tasks, + "executedTasks": plan.rerun_tasks, + }), + &[], + ), + )?; + for task in &plan.repair_plan.materialized_tasks { + self.trace( + TraceEvent::new( + SpanKind::Task, + TracePhase::Completed, + "task.reused", + &trace_id, + &run_id, + self.clock.now(), + ) + .task(&task.task_id) + .attributes( + serde_json::json!({ + "disposition": "reused", + "sourceRunId": task.source_run_id, + "sourceTaskId": task.source_task_id, + "sourceAttempt": task.source_attempt, + "outputDigest": task.metadata.output_digest, + }), + &[], + ), + )?; + } + let outcome = self + .drive(&run_id, &trace_id, options, cancellation) + .await?; + let artifacts = self + .store + .list_tasks(&outcome.run_id)? + .into_iter() + .flat_map(|task| task.artifact_manifest) + .map(|artifact| (artifact.path.clone(), artifact)) + .collect::>() + .into_values() + .collect(); + Ok(RetryOutcome { + run_id: outcome.run_id, + source_run_id: source.run_id, + trace_id: outcome.trace_id, + state: outcome.state, + failed_only: plan.failed_only, + retry_roots: plan.retry_roots, + reused_tasks: plan.reused_tasks, + executed_tasks: plan.rerun_tasks, + artifacts, + output: outcome.output, + }) + } + #[allow(clippy::too_many_arguments)] pub async fn repair( &self, @@ -4866,7 +5128,8 @@ const fn retryable_error(error: &RuntimeError) -> bool { | RuntimeError::ExternalEffectUncertain(_) | RuntimeError::Cancelled | RuntimeError::Json(_) - | RuntimeError::RepairBlocked { .. } => false, + | RuntimeError::RepairBlocked { .. } + | RuntimeError::RetryBlocked { .. } => false, } } @@ -5030,6 +5293,38 @@ mod tests { } } + #[derive(Default)] + struct TerminalRetryProvider(AtomicU64); + + #[async_trait] + impl ModelProvider for TerminalRetryProvider { + fn name(&self) -> &'static str { + "fake" + } + + async fn complete( + &self, + _request: &ProviderRequest, + _cancellation: &CancellationToken, + ) -> Result { + if self.0.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(ProviderError::Malformed( + "transient terminal retry fixture".to_owned(), + )); + } + let text = r#"{"value":"recovered"}"#.to_owned(); + Ok(ProviderResponse { + response_id: Some("retry-recovered".to_owned()), + text: text.clone(), + tool_calls: Vec::new(), + assistant_content: vec![ContentBlock::Text { text }], + continuation: None, + usage: Usage::default(), + finish_reason: FinishReason::Complete, + }) + } + } + #[derive(Default)] struct SelectiveRepairProvider { first_calls: AtomicU64, @@ -7023,6 +7318,238 @@ spec: })); } + #[tokio::test] + async fn terminal_failed_only_retry_reuses_success_and_replays_offline() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(TerminalRetryProvider::default()); + let runtime = Runtime::new(store.clone(), directory.path()) + .with_clock(Arc::new(FixedClock)) + .with_ids(Arc::new(SequenceIds::default())) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: terminal-retry } +spec: + providers: + fake: { kind: fake } + agents: + recover: + provider: fake + model: fake + instructions: return a recovery object + structuredOutput: + type: object + required: [value] + additionalProperties: false + properties: + value: { type: string } + actions: + assign: { kind: builtin.assign } + tasks: + - { id: first, uses: "action:assign", with: { value: durable } } + - { id: second, uses: "agent:recover", needs: [first], with: { prompt: recover } } + - { id: third, uses: "action:assign", needs: [second], with: { value: "${{ tasks.second.output.value }}" } } +"#; + let (workflow, compiled) = compile_fixture(source); + let source_run_id = match runtime + .start( + &workflow, + &compiled, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected terminal source failure, got {other:?}"), + }; + let source_before = store.load_run(&source_run_id).expect("source"); + let plan = runtime + .plan_retry(&source_run_id, &workflow, &compiled, &[], true, false) + .expect("retry plan"); + assert!(plan.compatible, "{:?}", plan.blocked_reuse); + assert_eq!(plan.retry_roots, ["second"]); + assert_eq!(plan.reused_tasks, ["first"]); + assert_eq!(plan.rerun_tasks, ["second", "third"]); + assert_eq!( + serde_json::to_value(&plan).expect("json")["failedOnly"], + true + ); + + let outcome = runtime + .retry( + &workflow, + &compiled, + plan, + Some("retry transient provider failure"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("retry"); + assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!(outcome.reused_tasks, ["first"]); + assert_eq!(outcome.executed_tasks, ["second", "third"]); + assert_eq!(provider.0.load(Ordering::SeqCst), 2); + let retry = store.load_run(&outcome.run_id).expect("retry run"); + assert_eq!(retry.mode, RunMode::Retry); + assert_eq!(retry.source_run_id.as_deref(), Some(source_run_id.as_str())); + assert_eq!(retry.retry_roots, ["second"]); + assert!(retry.retry_failed_only); + assert_eq!( + store.load_run(&source_run_id).expect("source unchanged"), + source_before + ); + let tasks = store.list_tasks(&outcome.run_id).expect("retry tasks"); + assert_eq!(tasks[0].disposition, TaskDisposition::Reused); + assert_eq!(tasks[1].disposition, TaskDisposition::Executed); + assert_eq!(tasks[1].attempt, 1); + + let replay = runtime + .replay(&outcome.run_id) + .await + .expect("offline replay"); + assert_eq!(replay.state, RunState::Succeeded); + assert!( + store + .list_effects(&replay.run_id) + .expect("effects") + .is_empty() + ); + assert_eq!(replay.output, outcome.output); + } + + #[tokio::test] + async fn retry_planning_enforces_identity_roots_acknowledgement_and_reconciliation() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: retry-planning } +spec: + actions: + assign: { kind: builtin.assign } + assert: { kind: builtin.assert } + tasks: + - { id: first, uses: "action:assign", with: { value: durable } } + - { id: second, uses: "action:assert", needs: [first], with: { that: false } } + - { id: third, uses: "action:assign", needs: [second], with: { value: done } } +"#; + let (workflow, compiled) = compile_fixture(source); + let source_run_id = match runtime + .start( + &workflow, + &compiled, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let successful_without_ack = runtime + .plan_retry( + &source_run_id, + &workflow, + &compiled, + &["first".to_owned()], + false, + false, + ) + .expect("blocked successful root"); + assert!(!successful_without_ack.compatible); + assert!( + successful_without_ack + .blocked_reuse + .iter() + .any(|block| { block.rule == "successful_root_requires_acknowledgement" }) + ); + let multiple = runtime + .plan_retry( + &source_run_id, + &workflow, + &compiled, + &["first".to_owned(), "second".to_owned()], + false, + true, + ) + .expect("multiple roots"); + assert!(multiple.compatible, "{:?}", multiple.blocked_reuse); + assert_eq!(multiple.retry_roots, ["first", "second"]); + + let changed = source.replace("value: durable", "value: changed"); + let (changed_workflow, changed_plan) = compile_fixture(&changed); + let mismatch = runtime + .plan_retry( + &source_run_id, + &changed_workflow, + &changed_plan, + &[], + true, + false, + ) + .expect("mismatch plan"); + assert!(!mismatch.compatible); + assert!( + mismatch + .blocked_reuse + .iter() + .any(|block| { block.rule == "retry_workflow_definition_mismatch" }) + ); + + let uncertain = EffectRequest::new( + &source_run_id, + "second", + 1, + 99, + "external.publish", + EffectClass::ExternalMutate, + Risk::High, + Idempotency::Unknown, + serde_json::json!({"record": "x"}), + "publish record", + "trace-retry", + ); + store + .record_effect_request(&uncertain, FixedClock.now()) + .expect("effect"); + store + .mark_effect_started(&uncertain.id, FixedClock.now()) + .expect("started"); + store + .mark_effect_uncertain(&uncertain.id, "unknown", FixedClock.now()) + .expect("uncertain"); + let blocked = runtime + .plan_retry(&source_run_id, &workflow, &compiled, &[], true, false) + .expect("blocked effect"); + assert!(!blocked.compatible); + assert!( + blocked + .blocked_reuse + .iter() + .any(|block| block.rule == "unreconciled_effect") + ); + store + .reconcile_effect_not_applied( + &uncertain.id, + "operator", + "remote lookup found no record", + FixedClock.now(), + ) + .expect("reconcile"); + let compatible = runtime + .plan_retry(&source_run_id, &workflow, &compiled, &[], true, false) + .expect("compatible after reconciliation"); + assert!(compatible.compatible, "{:?}", compatible.blocked_reuse); + } + #[tokio::test] async fn repair_blocks_missing_state_delta_and_tampered_reused_output_digest() { let directory = tempdir().expect("tempdir"); diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index 3cea8ab..0d314fb 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -19,7 +19,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use thiserror::Error; -pub const DATABASE_SCHEMA_VERSION: u32 = 8; +pub const DATABASE_SCHEMA_VERSION: u32 = 9; pub const RUNTIME_STATE_VERSION: u32 = 1; pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; pub const AUDIT_EVENT_VERSION: u32 = 1; @@ -293,6 +293,13 @@ CREATE INDEX idx_effect_reconciliations_run_created ON effect_reconciliations(run_id, created_at, reconciliation_id); "#; +const MIGRATION_9: &str = r#" +ALTER TABLE runs ADD COLUMN retry_roots_json TEXT; +ALTER TABLE runs ADD COLUMN retry_reason TEXT; +ALTER TABLE runs ADD COLUMN retry_format_version INTEGER; +ALTER TABLE runs ADD COLUMN retry_failed_only INTEGER NOT NULL DEFAULT 0; +"#; + #[derive(Clone)] pub struct SqliteStore { connection: Arc>, @@ -348,6 +355,10 @@ pub struct RunRecord { pub repair_roots: Vec, pub repair_reason: Option, pub repair_format_version: Option, + pub retry_roots: Vec, + pub retry_reason: Option, + pub retry_format_version: Option, + pub retry_failed_only: bool, pub base_path: Option, pub cancellation_requested: bool, pub created_at: DateTime, @@ -362,6 +373,7 @@ pub enum RunMode { Replay, Fork, Repair, + Retry, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -965,11 +977,159 @@ impl SqliteStore { Ok(()) } + #[allow(clippy::too_many_arguments)] + pub fn create_retry_run( + &self, + run_id: &str, + source_run_id: &str, + source_workflow_digest: &str, + workflow_schema_version: &str, + workflow: &Value, + plan: &CompiledPlan, + inputs: &Value, + working_memory: &Value, + retry_roots: &[String], + failed_only: bool, + reason: Option<&str>, + reused_tasks: &[ReusedTaskMaterialization], + task_decisions: &Value, + base_path: &Path, + now: DateTime, + trace_id: &str, + ) -> Result<(), StoreError> { + let reused = reused_tasks + .iter() + .map(|task| (task.task_id.as_str(), task)) + .collect::>(); + let _artifact_guard = self.artifact_lock.lock(); + let _artifact_file_lock = self.artifact_store.lock_exclusive()?; + for task in reused_tasks { + verify_artifact_manifest( + self.artifact_store.as_ref(), + &task.metadata.artifact_manifest, + )?; + } + let mut connection = self.connection.lock(); + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO runs (run_id, runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, state, mode, source_run_id, source_workflow_digest, retry_roots_json, retry_reason, retry_format_version, retry_failed_only, base_path, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, 1, ?17, ?18, ?19, ?19)", + params![ + run_id, + RUNTIME_STATE_VERSION, + plan.workflow_digest, + workflow_schema_version, + plan.plan_digest, + plan.format_version, + encode(workflow)?, + encode(plan)?, + encode(inputs)?, + encode(working_memory)?, + encode_enum(RunState::Running)?, + encode_enum(RunMode::Retry)?, + source_run_id, + source_workflow_digest, + encode(retry_roots)?, + reason, + failed_only, + base_path.display().to_string(), + now.to_rfc3339(), + ], + )?; + for (position, task_id) in plan.order.iter().enumerate() { + let position = i64::try_from(position).map_err(|_| { + StoreError::Incompatible("task position exceeds SQLite integer range".to_owned()) + })?; + if let Some(task) = reused.get(task_id.as_str()) { + transaction.execute( + "INSERT INTO task_states (run_id, task_id, position, state, attempt, output_json, disposition, metadata_version, source_run_id, source_task_id, source_attempt, definition_fingerprint, input_digest, output_contract_fingerprint, output_digest, state_delta_json, state_delta_digest, artifact_manifest_json, reuse_decision_json, updated_at) VALUES (?1, ?2, ?3, ?4, 0, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", + params![ + run_id, + task_id, + position, + encode_enum(TaskState::Succeeded)?, + encode(&task.output)?, + encode_enum(TaskDisposition::Reused)?, + task.metadata.execution.metadata_version, + task.source_run_id, + task.source_task_id, + task.source_attempt, + task.metadata.execution.definition_fingerprint, + task.metadata.execution.input_digest, + task.metadata.execution.output_contract_fingerprint, + task.metadata.output_digest, + encode(&task.metadata.state_delta)?, + task.metadata.state_delta_digest, + encode(&task.metadata.artifact_manifest)?, + encode(&task.reuse_decision)?, + now.to_rfc3339(), + ], + )?; + append_audit_tx( + &transaction, + run_id, + "retry.task_reused", + Some(task_id), + trace_id, + &serde_json::json!({ + "sourceRunId": task.source_run_id, + "sourceTaskId": task.source_task_id, + "sourceAttempt": task.source_attempt, + "outputDigest": task.metadata.output_digest, + "decision": task.reuse_decision, + }), + now, + )?; + record_artifact_references_tx( + &transaction, + run_id, + task_id, + &task.metadata.artifact_manifest, + Some(&task.source_run_id), + Some(&task.source_task_id), + now, + )?; + } else { + transaction.execute( + "INSERT INTO task_states (run_id, task_id, position, state, disposition, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + run_id, + task_id, + position, + encode_enum(TaskState::Pending)?, + encode_enum(TaskDisposition::Executed)?, + now.to_rfc3339(), + ], + )?; + } + } + append_audit_tx( + &transaction, + run_id, + "retry.created", + None, + trace_id, + &serde_json::json!({ + "sourceRunId": source_run_id, + "sourceWorkflowDigest": source_workflow_digest, + "workflowDigest": plan.workflow_digest, + "retryRoots": retry_roots, + "failedOnly": failed_only, + "reason": reason, + "reusedTasks": reused_tasks.iter().map(|task| task.task_id.as_str()).collect::>(), + "taskDecisions": task_decisions, + }), + now, + )?; + checkpoint_tx(&transaction, run_id, now)?; + transaction.commit()?; + Ok(()) + } + pub fn load_run(&self, run_id: &str) -> Result { let connection = self.connection.lock(); connection .query_row( - "SELECT runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, output_json, state, mode, parent_run_id, cancellation_requested, created_at, updated_at, base_path, source_run_id, source_workflow_digest, repair_roots_json, repair_reason, repair_format_version FROM runs WHERE run_id = ?1", + "SELECT runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, output_json, state, mode, parent_run_id, cancellation_requested, created_at, updated_at, base_path, source_run_id, source_workflow_digest, repair_roots_json, repair_reason, repair_format_version, retry_roots_json, retry_reason, retry_format_version, retry_failed_only FROM runs WHERE run_id = ?1", [run_id], |row| { let state_version: u32 = row.get(0)?; @@ -997,6 +1157,10 @@ impl SqliteStore { row.get::<_, Option>(19)?, row.get::<_, Option>(20)?, row.get::<_, Option>(21)?, + row.get::<_, Option>(22)?, + row.get::<_, Option>(23)?, + row.get::<_, Option>(24)?, + row.get::<_, bool>(25)?, )) }, ) @@ -1037,6 +1201,14 @@ impl SqliteStore { .unwrap_or_default(), repair_reason: row.20, repair_format_version: row.21, + retry_roots: row + .22 + .map(|value| decode(&value, "run.retry_roots")) + .transpose()? + .unwrap_or_default(), + retry_reason: row.23, + retry_format_version: row.24, + retry_failed_only: row.25, base_path: row.16, cancellation_requested: row.13, created_at: parse_time(&row.14, "created_at")?, @@ -2938,6 +3110,7 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { (6_u32, MIGRATION_6), (7_u32, MIGRATION_7), (8_u32, MIGRATION_8), + (9_u32, MIGRATION_9), ]; for (version, sql) in migrations .into_iter() @@ -3408,6 +3581,7 @@ spec: MIGRATION_5, MIGRATION_6, MIGRATION_7, + MIGRATION_8, ] .into_iter() .enumerate() @@ -3980,6 +4154,53 @@ spec: ); } + #[test] + fn creates_retry_lineage_separately_from_repair_metadata() { + let store = SqliteStore::open_memory().expect("store"); + create(&store, "source"); + let (workflow, plan) = fixture(); + store + .create_retry_run( + "retry", + "source", + &plan.workflow_digest, + API_VERSION, + &workflow, + &plan, + &serde_json::json!({}), + &serde_json::json!({}), + &["one".to_owned()], + true, + Some("retry failed tasks"), + &[], + &serde_json::json!([]), + Path::new("."), + Utc::now(), + "trace-retry", + ) + .expect("create retry"); + + let retry = store.load_run("retry").expect("retry run"); + assert_eq!(retry.mode, RunMode::Retry); + assert_eq!(retry.source_run_id.as_deref(), Some("source")); + assert_eq!(retry.retry_roots, ["one"]); + assert_eq!(retry.retry_reason.as_deref(), Some("retry failed tasks")); + assert_eq!(retry.retry_format_version, Some(1)); + assert!(retry.retry_failed_only); + assert!(retry.repair_roots.is_empty()); + assert_eq!( + store.list_tasks("retry").expect("tasks")[0].disposition, + TaskDisposition::Executed + ); + assert!( + store + .audit_events("retry") + .expect("audit") + .iter() + .any(|event| event.event_type == "retry.created") + ); + } + #[test] fn interrupted_repair_migration_can_restart_cleanly() { let directory = tempdir().expect("temp dir"); diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index bb3343e..c64118a 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -6,9 +6,10 @@ SQLite is the local history and correctness boundary. Run, task, effect, approva - Resume continues the same run from durable task state. Confirmed effects are reused. A requested-but-not-started effect may execute; a started-but-unconfirmed effect fails as uncertain. - Recorded replay creates a replay record from terminal stored outputs and calls no provider, tool, network, process, or filesystem executor. +- Terminal retry creates a new source-linked run for an identical workflow, materializes compatible successful boundaries, and executes failed or explicitly selected roots plus their descendants with fresh attempts. - Selective repair creates a new source-linked run, materializes compatible successful task outputs and committed state deltas, then executes selected roots and descendants with fresh effects from a target workflow. - Fork creates a new run linked to the old run and intentionally permits fresh effects. -- Retry creates a new task attempt only within the task’s explicit bound. An unsafe unresolved effect is not retried. +- A task's `retry` policy creates another attempt inside the same run only within its explicit bound. An unsafe unresolved effect is not retried. An effect ID is SHA-256 over run ID, task ID, task attempt, ordinal, operation, and input digest. Each record carries its format version, idempotency key, effect class, risk, status, request/result or error, timestamps, trace correlation, and confirmation flag. The request commits before the executor starts. This supports deterministic reuse of completed results but does not prove exactly-once behavior in an external system. @@ -18,7 +19,7 @@ Working-memory replacement, the task transition, checkpoint, and audit event com Successful workspace mutations are ingested into the local content-addressed artifact store before task completion. Ingestion uses an atomic temporary file, SHA-256 identity, immutable deduplicated blobs, a cross-process lock, and a durable one-hour lease. Successful task completion then commits the artifact references with the definition fingerprint, resolved-input digest, output-contract fingerprint, output digest, immutable state delta and digest, audit event, and checkpoint, and releases the ingestion lease in the same SQLite transaction. Repair initialization starts from target initial memory and applies only reused successful task deltas in topological order. It never copies a terminal source's final memory snapshot. -Repair planning is effect-free. A source task is reusable only when its metadata version, definition, dependencies, resolved inputs, output contract/value, state delta, content-addressed artifacts, and effect certainty are compatible. The repair run stores the reused result, artifact references, and provenance in its own rows, so later source-row or workspace deletion does not break it. Missing or corrupt CAS bytes block reuse before a repair run is created. +Retry and repair planning are effect-free. A source task is reusable only when its metadata version, definition, dependencies, resolved inputs, output contract/value, state delta, content-addressed artifacts, and effect certainty are compatible. Retry additionally requires the exact stored workflow digest; changed definitions require repair. The new run stores the reused result, artifact references, and provenance in its own rows, so later source-row or workspace deletion does not break it. Missing or corrupt CAS bytes block reuse before a run is created. Cancellation is both an injected token and a durable run flag. CLI SIGINT and SIGTERM cancel in-flight async calls and return exit `130`; `agentctl cancel` records a request for another process to observe. An overall CLI deadline can be set with `--timeout-seconds`, in addition to task/tool/provider/protocol bounds. A provider, tool, process, MCP, or A2A timeout/cancellation/transport loss after dispatch marks the effect `uncertain`; resume refuses to guess and requires reconciliation. An applied reconciliation supplies a validated recorded result. A not-applied or compensated reconciliation resumes with a fresh task and effect attempt. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 62cb649..f63caf4 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -48,7 +48,7 @@ These are useful extensions but are not required by the product thesis. They nee - Successful tasks from databases created before schema 5 require explicit `runs analyze`/`runs upgrade`. Only provable metadata is imported; unprovable boundaries are returned as conservative safe repair roots. - Automatic artifact ingestion covers regular files up to 16 MiB reported by successful built-in workspace-mutation results. Larger outputs and artifacts produced only by opaque external effects require an explicit bounded import/export integration. The local CAS must be backed up with SQLite; missing or corrupt blob bytes block repair before run creation and report the expected artifact identity. - An applied non-idempotent mutation in a repair closure remains blocked from duplicate execution unless a confirmed compensation is linked. Reconciliation supports immutable `applied`, `not_applied`, and `compensated` records, validated results, policy authorization, and operation-specific verification hooks; it does not provide exactly-once delivery. -- Retry remains a bounded same-run task policy. There is no separate command that creates a new terminal-source retry run for an unchanged workflow; use repair with an unchanged target definition and explicit roots when its compatibility checks fit. +- Terminal retry requires an identical workflow digest and a terminal source. It creates a new source-linked run, reuses only proven compatible successful boundaries, and freshly executes the selected closure. Use repair for a changed definition, resume for a non-terminal run, replay for effect-free reconstruction, and fork for knowingly broad fresh execution. - Tool-using OpenAI/Azure agents require stored-response continuation. `store: false` is rejected until stateless response-item replay is implemented. - Anthropic, Google, Azure OpenAI, MCP, and A2A are native and mock-tested in this release, not live-tested. Only the OpenAI GPT-5.6 tool path has live end-to-end evidence. - The current local OCI runtime, vulnerability-scan, and SBOM evidence is Linux arm64. Linux x64 is configured in the unpushed Ubuntu workflow but has not executed. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 5e86143..faeab91 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -71,7 +71,7 @@ cargo xtask acceptance-container | Artifact CAS | 19 store tests and 38 runtime tests | CLI acceptance and hardened OCI acceptance passed | verified | | Legacy upgrades | all retained schema fixtures, dry-run, rollback, import, boundary, repair/replay tests | migration verification command added; full composite rerun pending | verified | | Reconciliation | immutable transition matrix, schema/tool/hook/policy, repair and resume tests | full composite rerun pending | verified | -| Terminal retry | pending | pending | open | +| Terminal retry | runtime/store identity, roots, acknowledgements, reconciliation, lineage, source immutability, and replay tests passed | packaged CLI scenario 30 and the 12-stage verification gate passed | verified | | Parallel/dynamic workflows | pending | pending | open | | Conditions/loops/sub-workflows | pending | pending | open | | Compensation/handoffs/streaming | pending | pending | open | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index f049696..b84fdcd 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -37,7 +37,7 @@ complete, every entry must have exactly one final disposition: | ART-001 | Durable artifacts | verified | implemented | | MIG-001 | Legacy selective repair | verified | implemented | | EFX-001 | Effect reconciliation | verified | implemented | -| RET-001 | Terminal-run retry | open | implemented | +| RET-001 | Terminal-run retry | verified | implemented | | ENC-001 | Sensitive-state encryption | open | implemented | | SEC-001 | Secret providers | open | implemented | | NET-001 | Network policy | open | implemented | @@ -154,8 +154,8 @@ complete, every entry must have exactly one final disposition: ### RET-001: Terminal-run retry -- Current behavior: task retry is same-run and bounded; terminal rerun requires - repair or a broad fork. +- Current behavior: task retry is same-run and bounded; `agentctl retry` + creates a distinct source-linked run for an unchanged terminal workflow. - User impact: operational retry of a failed unchanged workflow is obscure. - Security or durability impact: a fork can repeat successful external effects. - Product decision: add a distinct source-linked retry plan and run mode. It @@ -170,7 +170,11 @@ complete, every entry must have exactly one final disposition: - Examples: durable pipeline retry after deterministic downstream failure. - Live evidence: bounded deterministic provider failure followed by live retry. - Documentation: retry versus resume, repair, replay, and fork. -- Final disposition: pending implementation evidence. +- Final disposition: implemented and verified by failed-only and selected-root + planning, multiple-root and successful-root acknowledgement tests, exact + workflow identity enforcement, reuse and source-immutability checks, + uncertain-effect reconciliation coverage, schema-9 lineage persistence, + offline replay, and packaged CLI acceptance. ### ENC-001: Envelope encryption for sensitive persisted fields diff --git a/docs/generated/CLI.md b/docs/generated/CLI.md index 1f78bea..3e4aff7 100644 --- a/docs/generated/CLI.md +++ b/docs/generated/CLI.md @@ -17,6 +17,7 @@ Commands: replay Reconstruct a terminal run only from recorded state and results fork Create a new run from a prior workflow with fresh effects repair Create a new run that reuses compatible upstream results and executes a repaired suffix + retry Retry failed or selected boundaries of an identical terminal workflow runs Analyze or upgrade retained legacy run records for selective reuse cancel Durably request cancellation inspect Inspect durable run, task, and audit state @@ -192,6 +193,34 @@ Options: -h, --help Print help ``` +## `agentctl retry` + +```text +Retry failed or selected boundaries of an identical terminal workflow + +Usage: agentctl retry [OPTIONS] + +Arguments: + + + +Options: + --failed + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --from + --plan + --verbose + --restart-successful + --reason + --db [default: .agentctl/runtime.db] + --interactive + --diff + --workspace + --timeout-seconds + -h, --help Print help +``` + ## `agentctl runs` ```text diff --git a/docs/guides/TERMINAL_RETRY.md b/docs/guides/TERMINAL_RETRY.md new file mode 100644 index 0000000..3df28d4 --- /dev/null +++ b/docs/guides/TERMINAL_RETRY.md @@ -0,0 +1,84 @@ +# Retry a terminal workflow + +Use terminal retry when a run is terminal, its workflow definition is unchanged, and failed or selected task boundaries should execute again. Retry creates a new source-linked run. It never reopens or mutates the source. + +## Plan failed boundaries + +Inspect the source and understand any failed, started, or uncertain effects: + +```console +agentctl inspect SOURCE_RUN_ID --db .agentctl/runtime.db --output json +agentctl effects --db .agentctl/runtime.db list SOURCE_RUN_ID --output json +``` + +Create an effect-free plan for every failed task and its descendants: + +```console +agentctl retry workflow.yaml SOURCE_RUN_ID \ + --failed \ + --plan \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` + +The plan reports retry roots, reusable successful tasks, tasks that will run, possible effects and approvals, compatibility blocks, and warnings. A compatible plan exits `0`; a blocked but parseable plan exits `3`. Retry requires the target workflow digest to equal the source digest exactly. Use repair if the workflow changed. + +Select one or more explicit roots instead of all failed tasks when needed: + +```console +agentctl retry workflow.yaml SOURCE_RUN_ID \ + --from publish \ + --from verify \ + --plan +``` + +`--failed` and `--from` are mutually exclusive. Selecting a task that already succeeded requires `--restart-successful`, because its closure will execute fresh effects. + +## Execute the retry + +```console +agentctl retry workflow.yaml SOURCE_RUN_ID \ + --failed \ + --reason "transient dependency recovered" \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` + +The result contains a new run ID, source run ID, retry roots, reused tasks, freshly executed tasks, final state, artifacts, and workflow output. `inspect` identifies the new run mode as `retry`, preserves the source lineage and reason, and marks each task `reused` or `executed`. + +Successful tasks outside the retry closure are reused only after their definition, dependencies, resolved inputs, output contract and digest, committed state delta, artifact integrity, and effect certainty pass the same boundary checks used by repair. Reused tasks dispatch no provider, tool, process, network, or filesystem operation. Roots and descendants start fresh task and effect attempts. + +## Resolve uncertain effects + +Retry will not guess whether an ambiguous external mutation happened. Inspect the effect and append an authorized reconciliation only after checking external reality: + +```console +agentctl effects --db .agentctl/runtime.db inspect EFFECT_ID +agentctl effects --db .agentctl/runtime.db reconcile EFFECT_ID \ + --status not-applied \ + --reason "remote system confirms no mutation" \ + --actor operator-name +``` + +Then create a new plan. There is no force flag and no exactly-once claim. See [Effect reconciliation](EFFECT_RECONCILIATION.md). + +## Choose the right recovery operation + +| Operation | Use it when | Identity and effects | +| --- | --- | --- | +| `resume` | The source is paused or otherwise non-terminal. | Continues the same run and definition. | +| `retry` | The source is terminal and the workflow is identical. | New linked run; compatible success is reused; selected closure is fresh. | +| `repair` | The source is terminal and the workflow was corrected. | New linked run under compatibility checks; selected closure is fresh. | +| `replay` | Stored results must be reconstructed or audited. | New recorded run with zero fresh effects. | +| `fork` | A broad fresh execution is knowingly intended. | New linked run that permits fresh effects throughout. | + +After a successful retry, recorded replay remains offline: + +```console +env -u OPENAI_API_KEY agentctl replay RETRY_RUN_ID \ + --db .agentctl/runtime.db \ + --output json \ + --color never +``` diff --git a/docs/guides/repair-a-failed-workflow.md b/docs/guides/repair-a-failed-workflow.md index 65dc6e2..93ca8fa 100644 --- a/docs/guides/repair-a-failed-workflow.md +++ b/docs/guides/repair-a-failed-workflow.md @@ -189,4 +189,4 @@ Recorded replay has a new replay run ID but the same semantic outputs. It dispat | `new_task_outside_repair_closure` | A new unrelated task has no result. | Add it as a root or choose an earlier common boundary. | | `unreconciled_effect` | Fresh execution may duplicate a mutation. | Inspect and reconcile external reality first. | -`retry` remains a task's bounded same-definition attempt policy in v1alpha1; there is no separate terminal-run `retry` command yet. Use repair for a changed target definition and fork for a broader intentionally fresh execution. +Use [`agentctl retry`](TERMINAL_RETRY.md) instead when the workflow definition is unchanged and the intent is to rerun failed or explicitly selected boundaries of a terminal source. Use repair for a corrected definition and fork for a broader intentionally fresh execution. diff --git a/docs/reference/DATABASE.md b/docs/reference/DATABASE.md index 0d594b3..3110a85 100644 --- a/docs/reference/DATABASE.md +++ b/docs/reference/DATABASE.md @@ -1,10 +1,10 @@ # Runtime database and migrations -The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `8`. +The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `9`. ## Stored records -- runs, source workflow, compiled plan, inputs, output, mode, state, parent linkage, and repair source/root metadata +- runs, source workflow, compiled plan, inputs, output, mode, state, parent linkage, and repair/retry source/root metadata - task states, attempts, output, errors, disposition, source attempt, versioned fingerprints/digests, state delta, artifact manifest, and reuse decision - effects, request/result/error, confirmation, and uncertainty - immutable effect reconciliation history, operator authorization, evidence, validated results, supersession, and compensation linkage @@ -18,7 +18,7 @@ The local SQLite database and its sibling artifact root are history and part of Working memory is stored on the run and in checkpoints. Provider credentials are not stored. Other confidential content may be stored, including prompts, tool output, and remote artifacts. -Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. A repair transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete a repair run. +Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. Migration 9 adds retry roots/reason/version and failed-only selection. A repair or retry transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete the derived run. Artifact manifests contain logical path/name, media type, byte size, SHA-256 digest, and CAS-relative path. Blob bytes live under `/artifacts/sha256/`; identical content is stored once. A completed repair/replay receives its own references, so source-row and workspace deletion do not break it. diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 4642964..46c9b6a 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -15,7 +15,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 29; +const ACCEPTANCE_SCENARIOS: usize = 30; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -566,7 +566,7 @@ pub fn run(root: &Path) -> Result<()> { scenario( 16, - "explicit retry recovers a definitive transient provider failure", + "bounded same-run retry recovers a definitive transient provider failure", ); let retry_workflow = workspace.join("retry.yaml"); write(&retry_workflow, RETRY_WORKFLOW)?; @@ -980,6 +980,100 @@ pub fn run(root: &Path) -> Result<()> { "second", )?; + scenario( + 30, + "terminal retry reuses the successful prefix and creates distinct lineage", + ); + let terminal_retry_workflow = workspace.join("terminal-retry.yaml"); + write(&terminal_retry_workflow, TERMINAL_RETRY_WORKFLOW)?; + let terminal_retry_db = directory.path().join("terminal-retry.db"); + let source = json_with_code( + &binary, + &workspace, + &run_args( + &terminal_retry_workflow, + &terminal_retry_db, + &workspace, + &[], + ), + 4, + )?; + let source_id = string_at(&source, "/error/runId")?; + let retry_plan = successful_json( + &binary, + &workspace, + &strings([ + "retry", + path(&terminal_retry_workflow)?, + source_id, + "--failed", + "--plan", + "--db", + path(&terminal_retry_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&retry_plan, "/data/compatible", true)?; + ensure_eq(&retry_plan, "/data/failedOnly", true)?; + ensure_eq(&retry_plan, "/data/retryRoots/0", "work")?; + ensure_eq(&retry_plan, "/data/reusedTasks/0", "first")?; + ensure_eq(&retry_plan, "/data/rerunTasks/0", "work")?; + ensure_eq(&retry_plan, "/data/rerunTasks/1", "third")?; + let retried = successful_json( + &binary, + &workspace, + &strings([ + "retry", + path(&terminal_retry_workflow)?, + source_id, + "--failed", + "--reason", + "acceptance terminal retry", + "--db", + path(&terminal_retry_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&retried, "/data/state", "succeeded")?; + ensure_eq(&retried, "/data/sourceRunId", source_id)?; + ensure_eq(&retried, "/data/reusedTasks/0", "first")?; + ensure_eq(&retried, "/data/executedTasks/0", "work")?; + ensure_eq(&retried, "/data/executedTasks/1", "third")?; + let retry_run_id = string_at(&retried, "/data/runId")?; + let retry_inspect = inspect(&binary, &workspace, &terminal_retry_db, retry_run_id)?; + ensure_eq(&retry_inspect, "/data/run/mode", "retry")?; + ensure_eq(&retry_inspect, "/data/run/sourceRunId", source_id)?; + ensure_eq(&retry_inspect, "/data/run/retryFailedOnly", true)?; + ensure_eq(&retry_inspect, "/data/run/retryRoots/0", "work")?; + ensure_eq(&retry_inspect, "/data/tasks/0/disposition", "reused")?; + ensure_eq(&retry_inspect, "/data/tasks/1/disposition", "executed")?; + ensure_eq(&retry_inspect, "/data/tasks/2/disposition", "executed")?; + let source_inspect = inspect(&binary, &workspace, &terminal_retry_db, source_id)?; + ensure_eq(&source_inspect, "/data/run/state", "failed")?; + let replay = successful_json( + &binary, + &workspace, + &strings([ + "replay", + retry_run_id, + "--db", + path(&terminal_retry_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let replay_id = string_at(&replay, "/data/runId")?; + let replay_inspect = inspect(&binary, &workspace, &terminal_retry_db, replay_id)?; + ensure!(array_len(&replay_inspect, "/data/effects")? == 0); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } @@ -2765,6 +2859,33 @@ spec: with: { prompt: hello } "#; +const TERMINAL_RETRY_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: terminal-retry } +spec: + policy: + processAllowlist: [sh] + approval: never + actions: + assign: { kind: builtin.assign } + recover: + kind: builtin.shell.exec + command: /bin/sh + args: [-c, "if [ -f .terminal-retry-ready ]; then printf recovered; else touch .terminal-retry-ready; exit 1; fi"] + timeoutSeconds: 5 + tasks: + - id: first + uses: action:assign + with: { value: durable } + - id: work + uses: action:recover + needs: [first] + - id: third + uses: action:assign + needs: [work] + with: { value: recovered } +"#; + const OPENAI_AUTH_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 kind: Workflow metadata: { name: missing-auth } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c4476a6..09450b1 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -305,6 +305,7 @@ fn generated_cli_reference(binary: &Path) -> Result { &["replay"], &["fork"], &["repair"], + &["retry"], &["runs"], &["runs", "analyze"], &["runs", "upgrade"], @@ -685,6 +686,7 @@ fn verify_public_documentation(root: &Path) -> Result<()> { "docs/guides/FIRST_AGENT_WORKFLOW.md", "docs/guides/WORKFLOW_AUTHORING.md", "docs/guides/LOCAL_OPERATION.md", + "docs/guides/TERMINAL_RETRY.md", "docs/guides/repair-a-failed-workflow.md", "docs/guides/LEGACY_RUN_UPGRADE.md", "docs/guides/EFFECT_RECONCILIATION.md", From 1c74e76d372435d35672e7241c9bb48b640eac3b Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 16:37:51 +0530 Subject: [PATCH 09/44] feat: encrypt sensitive persisted state --- Cargo.lock | 153 ++- Cargo.toml | 3 + README.md | 1 + crates/agentctl-cli/src/main.rs | 103 ++ crates/agentctl-store/Cargo.toml | 3 + crates/agentctl-store/src/encryption.rs | 448 +++++++ crates/agentctl-store/src/lib.rs | 1292 +++++++++++++++++-- docs/LIMITATIONS.md | 4 +- docs/SECURITY.md | 5 +- docs/THREAT_MODEL.md | 3 +- docs/execution/COMPLETENESS_VERIFICATION.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 12 +- docs/generated/CLI.md | 66 + docs/guides/SENSITIVE_STATE_ENCRYPTION.md | 60 + docs/reference/DATABASE.md | 11 +- docs/reference/ENVIRONMENT_AND_PATHS.md | 4 + fuzz/Cargo.lock | 152 ++- xtask/Cargo.toml | 1 + xtask/src/acceptance.rs | 177 ++- xtask/src/main.rs | 5 + 20 files changed, 2379 insertions(+), 125 deletions(-) create mode 100644 crates/agentctl-store/src/encryption.rs create mode 100644 docs/guides/SENSITIVE_STATE_ENCRYPTION.md diff --git a/Cargo.lock b/Cargo.lock index a1e2490..9436551 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,42 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "aes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", +] + +[[package]] +name = "aes-gcm" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", + "zeroize", +] + [[package]] name = "agentctl-cli" version = "0.2.0" @@ -121,7 +157,9 @@ dependencies = [ name = "agentctl-store" version = "0.2.0" dependencies = [ + "aes-gcm", "agentctl-core", + "base64", "chrono", "fs2", "hex", @@ -132,6 +170,7 @@ dependencies = [ "sha2", "tempfile", "thiserror", + "zeroize", ] [[package]] @@ -297,6 +336,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -368,6 +416,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout", +] + [[package]] name = "clap" version = "4.6.4" @@ -417,6 +476,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -429,6 +494,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -457,6 +528,35 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -487,8 +587,8 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", ] [[package]] @@ -759,6 +859,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "h2" version = "0.4.15" @@ -867,6 +976,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1065,6 +1183,15 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -1367,6 +1494,17 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2248,6 +2386,16 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -2658,6 +2806,7 @@ name = "xtask" version = "0.2.0" dependencies = [ "anyhow", + "base64", "hex", "nix", "rusqlite", diff --git a/Cargo.toml b/Cargo.toml index df2ba0e..6b92bef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,8 +22,10 @@ repository = "https://github.com/opensourceops/agentctl" readme = "README.md" [workspace.dependencies] +aes-gcm = { version = "0.11.0", features = ["zeroize"] } anyhow = "1.0.100" async-trait = "0.1.89" +base64 = "0.22.1" bytes = "1.11.0" chrono = { version = "0.4.42", default-features = false, features = ["clock", "serde"] } clap = { version = "4.5.53", features = ["derive", "env", "string"] } @@ -53,6 +55,7 @@ tokio-util = "0.7.17" url = { version = "2.5.7", features = ["serde"] } uuid = { version = "1.18.1", features = ["serde", "v7"] } wiremock = "0.6.5" +zeroize = "1.9.0" [workspace.lints.rust] unsafe_code = "forbid" diff --git a/README.md b/README.md index 2e7e5d4..32108dd 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md) and [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). +For confidential workflow history, use [Sensitive-state encryption](docs/guides/SENSITIVE_STATE_ENCRYPTION.md). ## Safety boundary diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index 3b0e572..a4f41ef 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -446,6 +446,36 @@ struct DbArgs { enum DbCommand { Stats, Migrate, + Encryption { + #[command(subcommand)] + command: DbEncryptionCommand, + }, +} + +#[derive(Debug, Subcommand)] +enum DbEncryptionCommand { + /// Inventory protected fields without exposing their values. + Inventory, + /// Transactionally encrypt every identified sensitive field. + Enable { + #[arg(long)] + key_id: String, + /// Environment variable containing a base64-encoded 32-byte key. + #[arg(long)] + key_env: String, + #[arg(long)] + dry_run: bool, + }, + /// Transactionally decrypt and re-encrypt every protected field with a new key. + Rotate { + #[arg(long)] + key_id: String, + /// Environment variable containing a base64-encoded 32-byte key. + #[arg(long)] + key_env: String, + #[arg(long)] + dry_run: bool, + }, } #[derive(Debug, Args)] @@ -1658,6 +1688,79 @@ fn db_command(output: OutputFormat, args: DbArgs) -> Result { format!("database schema is at version {}", store.schema_version()), )?; } + DbCommand::Encryption { command } => match command { + DbEncryptionCommand::Inventory => { + let inventory = store + .encryption_inventory() + .map_err(CliError::persistence)?; + print_value( + output, + "EncryptionInventory", + &inventory, + Vec::new(), + format!( + "state encryption: {}; protected={}, encrypted={}, plaintext={}, invalid={}", + if inventory.enabled { + format!( + "enabled key={} reference={}", + inventory.key_id.as_deref().unwrap_or("unknown"), + inventory.key_reference.as_deref().unwrap_or("unknown") + ) + } else { + "disabled".to_owned() + }, + inventory.protected_values, + inventory.encrypted_values, + inventory.plaintext_values, + inventory.invalid_envelopes, + ), + )?; + } + DbEncryptionCommand::Enable { + key_id, + key_env, + dry_run, + } => { + let report = store + .enable_encryption(&key_id, &key_env, dry_run, Utc::now()) + .map_err(CliError::persistence)?; + print_value( + output, + "EncryptionMigration", + &report, + Vec::new(), + format!( + "{} state encryption with key {}: scanned {}, rewrote {}", + if dry_run { "planned" } else { "enabled" }, + key_id, + report.values_scanned, + report.values_rewritten, + ), + )?; + } + DbEncryptionCommand::Rotate { + key_id, + key_env, + dry_run, + } => { + let report = store + .rotate_encryption_key(&key_id, &key_env, dry_run, Utc::now()) + .map_err(CliError::persistence)?; + print_value( + output, + "EncryptionMigration", + &report, + Vec::new(), + format!( + "{} state-encryption rotation to key {}: scanned {}, rewrote {}", + if dry_run { "planned" } else { "completed" }, + key_id, + report.values_scanned, + report.values_rewritten, + ), + )?; + } + }, } Ok(EXIT_OK) } diff --git a/crates/agentctl-store/Cargo.toml b/crates/agentctl-store/Cargo.toml index 9e096e1..6c92b2b 100644 --- a/crates/agentctl-store/Cargo.toml +++ b/crates/agentctl-store/Cargo.toml @@ -10,7 +10,9 @@ repository.workspace = true readme.workspace = true [dependencies] +aes-gcm.workspace = true agentctl-core = { version = "0.2.0", path = "../agentctl-core" } +base64.workspace = true chrono.workspace = true fs2.workspace = true hex.workspace = true @@ -21,6 +23,7 @@ serde_json.workspace = true sha2.workspace = true tempfile.workspace = true thiserror.workspace = true +zeroize.workspace = true [dev-dependencies] diff --git a/crates/agentctl-store/src/encryption.rs b/crates/agentctl-store/src/encryption.rs new file mode 100644 index 0000000..7212754 --- /dev/null +++ b/crates/agentctl-store/src/encryption.rs @@ -0,0 +1,448 @@ +use std::sync::Arc; + +use aes_gcm::aead::array::Array; +use aes_gcm::aead::{Aead, Generate, KeyInit, Payload}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; + +use super::StoreError; + +pub const ENCRYPTION_FORMAT_VERSION: u32 = 1; +pub const ENCRYPTION_ALGORITHM: &str = "AES-256-GCM"; +pub(crate) const ENVELOPE_PREFIX: &str = "agentctl.encrypted.v1:"; +const KEY_CHECK_VALUE: &str = "agentctl-state-key-check-v1"; +pub const KEY_CHECK_CONTEXT: &str = "state_encryption.key_check"; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct SensitiveColumn { + pub table: &'static str, + pub column: &'static str, +} + +impl SensitiveColumn { + pub fn context(self) -> String { + format!("{}.{}", self.table, self.column) + } +} + +pub(crate) const SENSITIVE_COLUMNS: &[SensitiveColumn] = &[ + SensitiveColumn { + table: "runs", + column: "workflow_json", + }, + SensitiveColumn { + table: "runs", + column: "plan_json", + }, + SensitiveColumn { + table: "runs", + column: "inputs_json", + }, + SensitiveColumn { + table: "runs", + column: "working_memory_json", + }, + SensitiveColumn { + table: "runs", + column: "output_json", + }, + SensitiveColumn { + table: "runs", + column: "repair_reason", + }, + SensitiveColumn { + table: "runs", + column: "retry_reason", + }, + SensitiveColumn { + table: "task_states", + column: "output_json", + }, + SensitiveColumn { + table: "task_states", + column: "error", + }, + SensitiveColumn { + table: "task_states", + column: "state_delta_json", + }, + SensitiveColumn { + table: "task_states", + column: "reuse_decision_json", + }, + SensitiveColumn { + table: "effects", + column: "input_json", + }, + SensitiveColumn { + table: "effects", + column: "expected_effect", + }, + SensitiveColumn { + table: "effects", + column: "result_json", + }, + SensitiveColumn { + table: "effects", + column: "error", + }, + SensitiveColumn { + table: "approvals", + column: "redacted_input_json", + }, + SensitiveColumn { + table: "approvals", + column: "expected_effect", + }, + SensitiveColumn { + table: "approvals", + column: "reason", + }, + SensitiveColumn { + table: "approvals", + column: "resolution_reason", + }, + SensitiveColumn { + table: "checkpoints", + column: "state_json", + }, + SensitiveColumn { + table: "audit_events", + column: "payload_json", + }, + SensitiveColumn { + table: "provider_sessions", + column: "continuation_json", + }, + SensitiveColumn { + table: "long_term_memory", + column: "value_json", + }, + SensitiveColumn { + table: "trace_events", + column: "event_json", + }, + SensitiveColumn { + table: "run_upgrades", + column: "analysis_json", + }, + SensitiveColumn { + table: "run_upgrades", + column: "upgraded_tasks_json", + }, + SensitiveColumn { + table: "effect_reconciliations", + column: "reason", + }, + SensitiveColumn { + table: "effect_reconciliations", + column: "evidence_json", + }, + SensitiveColumn { + table: "effect_reconciliations", + column: "result_json", + }, + SensitiveColumn { + table: "effect_reconciliations", + column: "result_schema_json", + }, + SensitiveColumn { + table: "effect_reconciliations", + column: "authorization_json", + }, +]; + +pub trait StateKeyResolver: Send + Sync { + /// Resolve a key reference to exactly 32 raw bytes. + /// + /// Implementations must not persist or log the returned value. + fn resolve(&self, reference: &str) -> Result>, StoreError>; +} + +#[derive(Debug, Default)] +pub struct EnvironmentKeyResolver; + +impl StateKeyResolver for EnvironmentKeyResolver { + fn resolve(&self, reference: &str) -> Result>, StoreError> { + validate_environment_reference(reference)?; + let encoded = Zeroizing::new(std::env::var(reference).map_err(|_| { + StoreError::Encryption(format!( + "state-encryption key environment reference `{reference}` is unavailable" + )) + })?); + let decoded = STANDARD.decode(encoded.as_bytes()).map_err(|_| { + StoreError::Encryption(format!( + "state-encryption key from `{reference}` must be base64" + )) + })?; + validate_key_bytes(reference, decoded) + } +} + +#[derive(Clone)] +pub(crate) enum StateProtection { + Plaintext, + Encrypted(EncryptionCodec), +} + +impl StateProtection { + pub fn is_enabled(&self) -> bool { + matches!(self, Self::Encrypted(_)) + } + + pub fn protect(&self, plaintext: &str, context: &str) -> Result { + match self { + Self::Plaintext => { + if is_encrypted_value(plaintext) { + return Err(StoreError::Encryption(format!( + "encrypted value for `{context}` has no configured state key" + ))); + } + Ok(plaintext.to_owned()) + } + Self::Encrypted(codec) => codec.encrypt(plaintext, context), + } + } + + pub fn expose(&self, stored: &str, context: &str) -> Result { + match self { + Self::Plaintext => { + if is_encrypted_value(stored) { + return Err(StoreError::Encryption(format!( + "encrypted value for `{context}` has no configured state key" + ))); + } + Ok(stored.to_owned()) + } + Self::Encrypted(codec) => { + if !is_encrypted_value(stored) { + return Err(StoreError::Encryption(format!( + "plaintext value found in protected field `{context}`" + ))); + } + codec.decrypt(stored, context) + } + } + } +} + +pub(crate) type SharedStateProtection = Arc>; + +#[derive(Clone)] +pub(crate) struct EncryptionCodec { + key_id: String, + key: Zeroizing>, +} + +impl EncryptionCodec { + pub fn resolve( + key_id: &str, + key_reference: &str, + resolver: &dyn StateKeyResolver, + ) -> Result { + validate_key_id(key_id)?; + let key = resolver.resolve(key_reference)?; + if key.len() != 32 { + return Err(StoreError::Encryption(format!( + "state-encryption key from `{key_reference}` must decode to exactly 32 bytes" + ))); + } + Ok(Self { + key_id: key_id.to_owned(), + key, + }) + } + + #[cfg(test)] + pub fn from_bytes(key_id: &str, key: Vec) -> Result { + validate_key_id(key_id)?; + let key = validate_key_bytes("test key", key)?; + Ok(Self { + key_id: key_id.to_owned(), + key, + }) + } + + pub fn key_id(&self) -> &str { + &self.key_id + } + + pub fn encrypt(&self, plaintext: &str, context: &str) -> Result { + let cipher = Aes256Gcm::new_from_slice(self.key.as_slice()) + .map_err(|_| StoreError::Encryption("invalid state-encryption key".to_owned()))?; + let nonce = Nonce::generate(); + let ciphertext = cipher + .encrypt( + &nonce, + Payload { + msg: plaintext.as_bytes(), + aad: context.as_bytes(), + }, + ) + .map_err(|_| StoreError::Encryption("state encryption failed".to_owned()))?; + let envelope = EncryptionEnvelope { + version: ENCRYPTION_FORMAT_VERSION, + algorithm: ENCRYPTION_ALGORITHM.to_owned(), + key_id: self.key_id.clone(), + nonce: STANDARD.encode(nonce.as_slice()), + ciphertext: STANDARD.encode(ciphertext), + }; + Ok(format!( + "{ENVELOPE_PREFIX}{}:{}", + self.key_id, + serde_json::to_string(&envelope)? + )) + } + + pub fn decrypt(&self, stored: &str, context: &str) -> Result { + let envelope = parse_envelope(stored, context)?; + if envelope.key_id != self.key_id { + return Err(StoreError::Encryption(format!( + "protected field `{context}` requires key ID `{}`, configured key ID is `{}`", + envelope.key_id, self.key_id + ))); + } + let nonce = STANDARD.decode(envelope.nonce.as_bytes()).map_err(|_| { + StoreError::Encryption(format!("protected field `{context}` has an invalid nonce")) + })?; + let nonce: [u8; 12] = nonce.try_into().map_err(|_| { + StoreError::Encryption(format!( + "protected field `{context}` has an invalid nonce length" + )) + })?; + let ciphertext = STANDARD + .decode(envelope.ciphertext.as_bytes()) + .map_err(|_| { + StoreError::Encryption(format!( + "protected field `{context}` has invalid ciphertext" + )) + })?; + let cipher = Aes256Gcm::new_from_slice(self.key.as_slice()) + .map_err(|_| StoreError::Encryption("invalid state-encryption key".to_owned()))?; + let nonce = Array(nonce); + let plaintext = cipher + .decrypt( + &nonce, + Payload { + msg: &ciphertext, + aad: context.as_bytes(), + }, + ) + .map_err(|_| { + StoreError::Encryption(format!( + "authentication failed for protected field `{context}`" + )) + })?; + String::from_utf8(plaintext).map_err(|_| { + StoreError::Encryption(format!("protected field `{context}` did not contain UTF-8")) + }) + } + + pub fn key_check(&self) -> Result { + self.encrypt(KEY_CHECK_VALUE, KEY_CHECK_CONTEXT) + } + + pub fn verify_key_check(&self, stored: &str) -> Result<(), StoreError> { + let value = self.decrypt(stored, KEY_CHECK_CONTEXT)?; + if value != KEY_CHECK_VALUE { + return Err(StoreError::Encryption( + "state-encryption key check is invalid".to_owned(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct EncryptionEnvelope { + version: u32, + algorithm: String, + key_id: String, + nonce: String, + ciphertext: String, +} + +pub(crate) fn is_encrypted_value(value: &str) -> bool { + value.starts_with(ENVELOPE_PREFIX) +} + +pub(crate) fn validate_envelope(value: &str, context: &str) -> Result<(), StoreError> { + parse_envelope(value, context).map(|_| ()) +} + +fn parse_envelope(value: &str, context: &str) -> Result { + let raw = value.strip_prefix(ENVELOPE_PREFIX).ok_or_else(|| { + StoreError::Encryption(format!("protected field `{context}` is not encrypted")) + })?; + let (prefix_key_id, raw) = raw.split_once(':').ok_or_else(|| { + StoreError::Encryption(format!( + "protected field `{context}` has a malformed encryption envelope" + )) + })?; + let envelope: EncryptionEnvelope = serde_json::from_str(raw).map_err(|_| { + StoreError::Encryption(format!( + "protected field `{context}` has a malformed encryption envelope" + )) + })?; + if envelope.version != ENCRYPTION_FORMAT_VERSION { + return Err(StoreError::Encryption(format!( + "protected field `{context}` uses unsupported encryption format {}", + envelope.version + ))); + } + if envelope.algorithm != ENCRYPTION_ALGORITHM { + return Err(StoreError::Encryption(format!( + "protected field `{context}` uses unsupported algorithm `{}`", + envelope.algorithm + ))); + } + validate_key_id(&envelope.key_id)?; + if prefix_key_id != envelope.key_id { + return Err(StoreError::Encryption(format!( + "protected field `{context}` has inconsistent key metadata" + ))); + } + Ok(envelope) +} + +fn validate_key_bytes(reference: &str, bytes: Vec) -> Result>, StoreError> { + if bytes.len() != 32 { + return Err(StoreError::Encryption(format!( + "state-encryption key from `{reference}` must decode to exactly 32 bytes" + ))); + } + Ok(Zeroizing::new(bytes)) +} + +fn validate_key_id(key_id: &str) -> Result<(), StoreError> { + if key_id.is_empty() + || key_id.len() > 128 + || !key_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + return Err(StoreError::Encryption( + "state-encryption key ID must contain 1-128 ASCII letters, digits, `.`, `_`, or `-`" + .to_owned(), + )); + } + Ok(()) +} + +fn validate_environment_reference(reference: &str) -> Result<(), StoreError> { + let mut bytes = reference.bytes(); + let first = bytes.next(); + if first.is_none_or(|byte| !(byte.is_ascii_uppercase() || byte == b'_')) + || !bytes.all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_') + { + return Err(StoreError::Encryption( + "state-encryption key environment reference must match [A-Z_][A-Z0-9_]*".to_owned(), + )); + } + Ok(()) +} diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index 0d314fb..97ef2c3 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -1,6 +1,7 @@ //! Versioned SQLite persistence for agentctl. pub mod artifact; +pub mod encryption; use std::collections::BTreeMap; use std::path::Path; @@ -12,14 +13,19 @@ use agentctl_core::state::{RunState, TaskState}; use agentctl_core::{CompiledPlan, PLAN_FORMAT_VERSION}; use artifact::{ArtifactStore, ArtifactStoreError, ArtifactVerification, LocalArtifactStore}; use chrono::{DateTime, Utc}; -use parking_lot::Mutex; +use encryption::{ + ENVELOPE_PREFIX, EncryptionCodec, EnvironmentKeyResolver, SENSITIVE_COLUMNS, + SharedStateProtection, StateKeyResolver, StateProtection, is_encrypted_value, + validate_envelope, +}; +use parking_lot::{Mutex, RwLock}; use rusqlite::{Connection, OptionalExtension, Transaction, TransactionBehavior, params}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value; use sha2::{Digest, Sha256}; use thiserror::Error; -pub const DATABASE_SCHEMA_VERSION: u32 = 9; +pub const DATABASE_SCHEMA_VERSION: u32 = 10; pub const RUNTIME_STATE_VERSION: u32 = 1; pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; pub const AUDIT_EVENT_VERSION: u32 = 1; @@ -300,11 +306,25 @@ ALTER TABLE runs ADD COLUMN retry_format_version INTEGER; ALTER TABLE runs ADD COLUMN retry_failed_only INTEGER NOT NULL DEFAULT 0; "#; +const MIGRATION_10: &str = r#" +CREATE TABLE state_encryption ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + format_version INTEGER NOT NULL, + key_id TEXT NOT NULL, + key_reference TEXT NOT NULL, + key_check TEXT NOT NULL, + maintenance INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL +); +"#; + #[derive(Clone)] pub struct SqliteStore { connection: Arc>, artifact_store: Arc, artifact_lock: Arc>, + protection: SharedStateProtection, + key_resolver: Arc, } #[derive(Debug, Error)] @@ -319,6 +339,8 @@ pub enum StoreError { Incompatible(String), #[error("durable state is corrupt: {0}")] Corrupt(String), + #[error("state encryption error: {0}")] + Encryption(String), #[error("run `{0}` was not found")] RunNotFound(String), #[error("task `{task_id}` was not found in run `{run_id}`")] @@ -642,8 +664,38 @@ pub struct DatabaseStats { pub effect_reconciliations: i64, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EncryptionInventory { + pub enabled: bool, + pub key_id: Option, + pub key_reference: Option, + pub protected_values: u64, + pub encrypted_values: u64, + pub plaintext_values: u64, + pub invalid_envelopes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EncryptionMigrationReport { + pub operation: String, + pub dry_run: bool, + pub key_id: String, + pub key_reference: String, + pub values_scanned: u64, + pub values_rewritten: u64, +} + impl SqliteStore { pub fn open(path: &Path) -> Result { + Self::open_with_key_resolver(path, Arc::new(EnvironmentKeyResolver)) + } + + pub fn open_with_key_resolver( + path: &Path, + key_resolver: Arc, + ) -> Result { if let Some(parent) = path.parent() && !parent.as_os_str().is_empty() { @@ -667,14 +719,32 @@ impl SqliteStore { let artifact_store = LocalArtifactStore::open(state_root.join("artifacts"))?; let _artifact_file_lock = artifact_store.lock_exclusive()?; recover_artifact_quarantine(&connection, &artifact_store)?; + let protection = load_state_protection(&connection, key_resolver.as_ref())?; + let inventory = encryption_inventory(&connection, &protection)?; + if protection.is_enabled() + && (inventory.plaintext_values != 0 || inventory.invalid_envelopes != 0) + { + return Err(StoreError::Encryption(format!( + "encrypted database contains {} plaintext and {} invalid protected value(s)", + inventory.plaintext_values, inventory.invalid_envelopes + ))); + } Ok(Self { connection: Arc::new(Mutex::new(connection)), artifact_store: Arc::new(artifact_store), artifact_lock: Arc::new(Mutex::new(())), + protection: Arc::new(RwLock::new(protection)), + key_resolver, }) } pub fn open_memory() -> Result { + Self::open_memory_with_key_resolver(Arc::new(EnvironmentKeyResolver)) + } + + pub fn open_memory_with_key_resolver( + key_resolver: Arc, + ) -> Result { let mut connection = Connection::open_in_memory()?; configure(&connection)?; migrate(&mut connection)?; @@ -682,6 +752,8 @@ impl SqliteStore { connection: Arc::new(Mutex::new(connection)), artifact_store: Arc::new(LocalArtifactStore::temporary()?), artifact_lock: Arc::new(Mutex::new(())), + protection: Arc::new(RwLock::new(StateProtection::Plaintext)), + key_resolver, }) } @@ -772,6 +844,141 @@ impl SqliteStore { .unwrap_or(0) } + pub fn encryption_inventory(&self) -> Result { + let connection = self.connection.lock(); + encryption_inventory(&connection, &self.protection.read()) + } + + pub fn enable_encryption( + &self, + key_id: &str, + key_reference: &str, + dry_run: bool, + now: DateTime, + ) -> Result { + if self.protection.read().is_enabled() { + return Err(StoreError::Encryption( + "state encryption is already enabled; use key rotation".to_owned(), + )); + } + let codec = EncryptionCodec::resolve(key_id, key_reference, self.key_resolver.as_ref())?; + let mut connection = self.connection.lock(); + let inventory = encryption_inventory(&connection, &StateProtection::Plaintext)?; + if inventory.encrypted_values != 0 || inventory.invalid_envelopes != 0 { + return Err(StoreError::Encryption( + "unencrypted database contains an unexpected encryption envelope".to_owned(), + )); + } + let report = EncryptionMigrationReport { + operation: "enable".to_owned(), + dry_run, + key_id: key_id.to_owned(), + key_reference: key_reference.to_owned(), + values_scanned: inventory.protected_values, + values_rewritten: if dry_run { + 0 + } else { + inventory.plaintext_values + }, + }; + if dry_run { + return Ok(report); + } + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + rewrite_sensitive_values( + &transaction, + &StateProtection::Plaintext, + &StateProtection::Encrypted(codec.clone()), + )?; + transaction.execute( + "INSERT INTO state_encryption (singleton, format_version, key_id, key_reference, key_check, updated_at) VALUES (1, 1, ?1, ?2, ?3, ?4)", + params![ + key_id, + key_reference, + codec.key_check()?, + now.to_rfc3339() + ], + )?; + transaction.commit()?; + *self.protection.write() = StateProtection::Encrypted(codec); + Ok(report) + } + + pub fn rotate_encryption_key( + &self, + key_id: &str, + key_reference: &str, + dry_run: bool, + now: DateTime, + ) -> Result { + let current = self.protection.read().clone(); + let StateProtection::Encrypted(current_codec) = ¤t else { + return Err(StoreError::Encryption( + "state encryption is not enabled".to_owned(), + )); + }; + if current_codec.key_id() == key_id { + return Err(StoreError::Encryption( + "rotation requires a different key ID".to_owned(), + )); + } + let next = EncryptionCodec::resolve(key_id, key_reference, self.key_resolver.as_ref())?; + let mut connection = self.connection.lock(); + let inventory = encryption_inventory(&connection, ¤t)?; + if inventory.plaintext_values != 0 || inventory.invalid_envelopes != 0 { + return Err(StoreError::Encryption( + "encrypted database is not in a fully protected state".to_owned(), + )); + } + let report = EncryptionMigrationReport { + operation: "rotate".to_owned(), + dry_run, + key_id: key_id.to_owned(), + key_reference: key_reference.to_owned(), + values_scanned: inventory.protected_values, + values_rewritten: if dry_run { + 0 + } else { + inventory.encrypted_values + }, + }; + if dry_run { + return Ok(report); + } + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let changed = transaction.execute( + "UPDATE state_encryption SET format_version = 1, key_id = ?1, key_reference = ?2, key_check = ?3, maintenance = 1, updated_at = ?4 WHERE singleton = 1", + params![ + key_id, + key_reference, + next.key_check()?, + now.to_rfc3339() + ], + )?; + if changed != 1 { + return Err(StoreError::Encryption( + "state-encryption configuration disappeared during rotation".to_owned(), + )); + } + rewrite_sensitive_values( + &transaction, + ¤t, + &StateProtection::Encrypted(next.clone()), + )?; + let changed = transaction.execute( + "UPDATE state_encryption SET maintenance = 0 WHERE singleton = 1 AND maintenance = 1", + [], + )?; + if changed != 1 { + return Err(StoreError::Encryption( + "state-encryption configuration disappeared during rotation".to_owned(), + )); + } + transaction.commit()?; + *self.protection.write() = StateProtection::Encrypted(next); + Ok(report) + } + #[allow(clippy::too_many_arguments)] pub fn create_run( &self, @@ -798,10 +1005,14 @@ impl SqliteStore { workflow_schema_version, plan.plan_digest, plan.format_version, - encode(workflow)?, - encode(plan)?, - encode(inputs)?, - encode(working_memory)?, + encode_protected(&self.protection, workflow, "runs.workflow_json")?, + encode_protected(&self.protection, plan, "runs.plan_json")?, + encode_protected(&self.protection, inputs, "runs.inputs_json")?, + encode_protected( + &self.protection, + working_memory, + "runs.working_memory_json" + )?, encode_enum(RunState::Running)?, encode_enum(mode)?, parent_run_id, @@ -826,8 +1037,9 @@ impl SqliteStore { trace_id, &serde_json::json!({"mode": mode, "planDigest": plan.plan_digest}), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -874,16 +1086,22 @@ impl SqliteStore { workflow_schema_version, plan.plan_digest, plan.format_version, - encode(workflow)?, - encode(plan)?, - encode(inputs)?, - encode(working_memory)?, + encode_protected(&self.protection, workflow, "runs.workflow_json")?, + encode_protected(&self.protection, plan, "runs.plan_json")?, + encode_protected(&self.protection, inputs, "runs.inputs_json")?, + encode_protected( + &self.protection, + working_memory, + "runs.working_memory_json" + )?, encode_enum(RunState::Running)?, encode_enum(RunMode::Repair)?, source_run_id, source_workflow_digest, encode(repair_roots)?, - reason, + reason + .map(|value| protect_text(&self.protection, value, "runs.repair_reason")) + .transpose()?, base_path.display().to_string(), now.to_rfc3339(), ], @@ -900,7 +1118,11 @@ impl SqliteStore { task_id, position, encode_enum(TaskState::Succeeded)?, - encode(&task.output)?, + encode_protected( + &self.protection, + &task.output, + "task_states.output_json" + )?, encode_enum(TaskDisposition::Reused)?, task.metadata.execution.metadata_version, task.source_run_id, @@ -910,10 +1132,18 @@ impl SqliteStore { task.metadata.execution.input_digest, task.metadata.execution.output_contract_fingerprint, task.metadata.output_digest, - encode(&task.metadata.state_delta)?, + encode_protected( + &self.protection, + &task.metadata.state_delta, + "task_states.state_delta_json" + )?, task.metadata.state_delta_digest, encode(&task.metadata.artifact_manifest)?, - encode(&task.reuse_decision)?, + encode_protected( + &self.protection, + &task.reuse_decision, + "task_states.reuse_decision_json" + )?, now.to_rfc3339(), ], )?; @@ -931,6 +1161,7 @@ impl SqliteStore { "decision": task.reuse_decision, }), now, + &self.protection, )?; record_artifact_references_tx( &transaction, @@ -971,8 +1202,9 @@ impl SqliteStore { "taskDecisions": task_decisions, }), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -1020,16 +1252,22 @@ impl SqliteStore { workflow_schema_version, plan.plan_digest, plan.format_version, - encode(workflow)?, - encode(plan)?, - encode(inputs)?, - encode(working_memory)?, + encode_protected(&self.protection, workflow, "runs.workflow_json")?, + encode_protected(&self.protection, plan, "runs.plan_json")?, + encode_protected(&self.protection, inputs, "runs.inputs_json")?, + encode_protected( + &self.protection, + working_memory, + "runs.working_memory_json" + )?, encode_enum(RunState::Running)?, encode_enum(RunMode::Retry)?, source_run_id, source_workflow_digest, encode(retry_roots)?, - reason, + reason + .map(|value| protect_text(&self.protection, value, "runs.retry_reason")) + .transpose()?, failed_only, base_path.display().to_string(), now.to_rfc3339(), @@ -1047,7 +1285,11 @@ impl SqliteStore { task_id, position, encode_enum(TaskState::Succeeded)?, - encode(&task.output)?, + encode_protected( + &self.protection, + &task.output, + "task_states.output_json" + )?, encode_enum(TaskDisposition::Reused)?, task.metadata.execution.metadata_version, task.source_run_id, @@ -1057,10 +1299,18 @@ impl SqliteStore { task.metadata.execution.input_digest, task.metadata.execution.output_contract_fingerprint, task.metadata.output_digest, - encode(&task.metadata.state_delta)?, + encode_protected( + &self.protection, + &task.metadata.state_delta, + "task_states.state_delta_json" + )?, task.metadata.state_delta_digest, encode(&task.metadata.artifact_manifest)?, - encode(&task.reuse_decision)?, + encode_protected( + &self.protection, + &task.reuse_decision, + "task_states.reuse_decision_json" + )?, now.to_rfc3339(), ], )?; @@ -1078,6 +1328,7 @@ impl SqliteStore { "decision": task.reuse_decision, }), now, + &self.protection, )?; record_artifact_references_tx( &transaction, @@ -1119,8 +1370,9 @@ impl SqliteStore { "taskDecisions": task_decisions, }), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -1184,11 +1436,24 @@ impl SqliteStore { workflow_digest: row.2, workflow_schema_version: row.3, plan_digest: row.4, - workflow: decode(&row.5, "workflow_json")?, - plan: decode(&row.6, "plan_json")?, - inputs: decode(&row.7, "inputs_json")?, - working_memory: decode(&row.8, "working_memory_json")?, - output: row.9.map(|value| decode(&value, "output_json")).transpose()?, + workflow: decode_protected( + &self.protection, + &row.5, + "runs.workflow_json", + )?, + plan: decode_protected(&self.protection, &row.6, "runs.plan_json")?, + inputs: decode_protected(&self.protection, &row.7, "runs.inputs_json")?, + working_memory: decode_protected( + &self.protection, + &row.8, + "runs.working_memory_json", + )?, + output: row + .9 + .map(|value| { + decode_protected(&self.protection, &value, "runs.output_json") + }) + .transpose()?, state: decode_enum(&row.10, "run.state")?, mode: decode_enum(&row.11, "run.mode")?, parent_run_id: row.12, @@ -1199,14 +1464,22 @@ impl SqliteStore { .map(|value| decode(&value, "run.repair_roots")) .transpose()? .unwrap_or_default(), - repair_reason: row.20, + repair_reason: row + .20 + .map(|value| { + expose_text(&self.protection, &value, "runs.repair_reason") + }) + .transpose()?, repair_format_version: row.21, retry_roots: row .22 .map(|value| decode(&value, "run.retry_roots")) .transpose()? .unwrap_or_default(), - retry_reason: row.23, + retry_reason: row + .23 + .map(|value| expose_text(&self.protection, &value, "runs.retry_reason")) + .transpose()?, retry_format_version: row.24, retry_failed_only: row.25, base_path: row.16, @@ -1264,6 +1537,7 @@ impl SqliteStore { "toolCalls": tool_calls, }), now, + &self.protection, )?; transaction.commit()?; Ok(()) @@ -1308,9 +1582,14 @@ impl SqliteStore { attempt: row.3, output: row .4 - .map(|value| decode(&value, "task.output")) + .map(|value| { + decode_protected(&self.protection, &value, "task_states.output_json") + }) + .transpose()?, + error: row + .5 + .map(|value| expose_text(&self.protection, &value, "task_states.error")) .transpose()?, - error: row.5, disposition: decode_enum(&row.7, "task.disposition")?, metadata_version: row.8, source_run_id: row.9, @@ -1322,7 +1601,9 @@ impl SqliteStore { output_digest: row.15, state_delta: row .16 - .map(|value| decode(&value, "task.state_delta")) + .map(|value| { + decode_protected(&self.protection, &value, "task_states.state_delta_json") + }) .transpose()?, state_delta_digest: row.17, artifact_manifest: row @@ -1332,7 +1613,13 @@ impl SqliteStore { .unwrap_or_default(), reuse_decision: row .19 - .map(|value| decode(&value, "task.reuse_decision")) + .map(|value| { + decode_protected( + &self.protection, + &value, + "task_states.reuse_decision_json", + ) + }) .transpose()?, updated_at: parse_time(&row.6, "task.updated_at")?, }) @@ -1371,12 +1658,32 @@ impl SqliteStore { .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; transaction.execute( "UPDATE task_states SET state = ?3, output_json = COALESCE(?4, output_json), error = ?5, attempt = attempt + ?7, updated_at = ?6 WHERE run_id = ?1 AND task_id = ?2", - params![run_id, task_id, encode_enum(next)?, output.map(encode).transpose()?, error, now.to_rfc3339(), i64::from(current == TaskState::Ready && next == TaskState::Running)], + params![ + run_id, + task_id, + encode_enum(next)?, + output + .map(|value| encode_protected( + &self.protection, + value, + "task_states.output_json" + )) + .transpose()?, + error + .map(|value| protect_text(&self.protection, value, "task_states.error")) + .transpose()?, + now.to_rfc3339(), + i64::from(current == TaskState::Ready && next == TaskState::Running) + ], )?; if let Some(memory) = working_memory { transaction.execute( "UPDATE runs SET working_memory_json = ?2, updated_at = ?3 WHERE run_id = ?1", - params![run_id, encode(memory)?, now.to_rfc3339()], + params![ + run_id, + encode_protected(&self.protection, memory, "runs.working_memory_json")?, + now.to_rfc3339() + ], )?; } else { transaction.execute( @@ -1392,8 +1699,9 @@ impl SqliteStore { trace_id, &serde_json::json!({"from": current, "to": next, "error": error}), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -1463,14 +1771,18 @@ impl SqliteStore { run_id, task_id, encode_enum(TaskState::Succeeded)?, - encode(output)?, + encode_protected(&self.protection, output, "task_states.output_json")?, encode_enum(TaskDisposition::Executed)?, metadata.execution.metadata_version, metadata.execution.definition_fingerprint, metadata.execution.input_digest, metadata.execution.output_contract_fingerprint, metadata.output_digest, - encode(&metadata.state_delta)?, + encode_protected( + &self.protection, + &metadata.state_delta, + "task_states.state_delta_json" + )?, metadata.state_delta_digest, encode(&metadata.artifact_manifest)?, now.to_rfc3339(), @@ -1492,7 +1804,11 @@ impl SqliteStore { if let Some(memory) = working_memory { transaction.execute( "UPDATE runs SET working_memory_json = ?2, updated_at = ?3 WHERE run_id = ?1", - params![run_id, encode(memory)?, now.to_rfc3339()], + params![ + run_id, + encode_protected(&self.protection, memory, "runs.working_memory_json")?, + now.to_rfc3339() + ], )?; } else { transaction.execute( @@ -1514,8 +1830,9 @@ impl SqliteStore { "stateDeltaDigest": metadata.state_delta_digest, }), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -1546,14 +1863,26 @@ impl SqliteStore { source.input_digest, source.output_contract_fingerprint, source.output_digest, - source.state_delta.as_ref().map(encode).transpose()?, + source + .state_delta + .as_ref() + .map(|value| encode_protected( + &self.protection, + value, + "task_states.state_delta_json" + )) + .transpose()?, source.state_delta_digest, encode(&source.artifact_manifest)?, - encode(&serde_json::json!({ - "recordedFromRunId": source.run_id, - "sourceDisposition": source.disposition, - "sourceProvenance": source.reuse_decision, - }))?, + encode_protected( + &self.protection, + &serde_json::json!({ + "recordedFromRunId": source.run_id, + "sourceDisposition": source.disposition, + "sourceProvenance": source.reuse_decision, + }), + "task_states.reuse_decision_json" + )?, now.to_rfc3339(), ], )?; @@ -1585,6 +1914,7 @@ impl SqliteStore { "outputDigest": source.output_digest, }), now, + &self.protection, )?; transaction.commit()?; Ok(()) @@ -1634,10 +1964,18 @@ impl SqliteStore { update.metadata.execution.input_digest, update.metadata.execution.output_contract_fingerprint, update.metadata.output_digest, - encode(&update.metadata.state_delta)?, + encode_protected( + &self.protection, + &update.metadata.state_delta, + "task_states.state_delta_json" + )?, update.metadata.state_delta_digest, encode(&update.metadata.artifact_manifest)?, - encode(&serde_json::json!({"legacyUpgrade": update.provenance}))?, + encode_protected( + &self.protection, + &serde_json::json!({"legacyUpgrade": update.provenance}), + "task_states.reuse_decision_json" + )?, now.to_rfc3339(), encode_enum(TaskState::Succeeded)?, ], @@ -1671,8 +2009,12 @@ impl SqliteStore { params![ run_id, upgrade_id, - encode(analysis)?, - encode(&upgraded_tasks)?, + encode_protected(&self.protection, analysis, "run_upgrades.analysis_json")?, + encode_protected( + &self.protection, + &upgraded_tasks, + "run_upgrades.upgraded_tasks_json" + )?, now.to_rfc3339(), ], )?; @@ -1689,8 +2031,9 @@ impl SqliteStore { "analysis": analysis, }), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -1719,7 +2062,18 @@ impl SqliteStore { .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; let changed = transaction.execute( "UPDATE runs SET state = ?2, output_json = COALESCE(?3, output_json), updated_at = ?4 WHERE run_id = ?1", - params![run_id, encode_enum(state)?, output.map(encode).transpose()?, now.to_rfc3339()], + params![ + run_id, + encode_enum(state)?, + output + .map(|value| encode_protected( + &self.protection, + value, + "runs.output_json" + )) + .transpose()?, + now.to_rfc3339() + ], )?; debug_assert_eq!(changed, 1); append_audit_tx( @@ -1730,8 +2084,9 @@ impl SqliteStore { trace_id, &serde_json::json!({"from": current, "to": state}), now, + &self.protection, )?; - checkpoint_tx(&transaction, run_id, now)?; + checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) } @@ -1758,8 +2113,12 @@ impl SqliteStore { encode_enum(request.idempotency)?, request.idempotency_key, request.input_digest, - encode(&request.input)?, - request.expected_effect, + encode_protected(&self.protection, &request.input, "effects.input_json")?, + protect_text( + &self.protection, + &request.expected_effect, + "effects.expected_effect" + )?, request.trace_id, encode_enum(EffectStatus::Requested)?, now.to_rfc3339(), @@ -1779,6 +2138,7 @@ impl SqliteStore { "risk": request.risk, }), now, + &self.protection, )?; transaction.commit()?; Ok(EffectRecord { @@ -1822,8 +2182,22 @@ impl SqliteStore { now: DateTime, ) -> Result<(), StoreError> { let (status, output, error, confirmed) = match result { - Ok(output) => (EffectStatus::Succeeded, Some(encode(output)?), None, true), - Err(error) => (EffectStatus::Failed, None, Some(error), false), + Ok(output) => ( + EffectStatus::Succeeded, + Some(encode_protected( + &self.protection, + output, + "effects.result_json", + )?), + None, + true, + ), + Err(error) => ( + EffectStatus::Failed, + None, + Some(protect_text(&self.protection, error, "effects.error")?), + false, + ), }; let changed = self.connection.lock().execute( "UPDATE effects SET status = ?2, result_json = ?3, error = ?4, confirmed = ?5, completed_at = ?6 WHERE effect_id = ?1 AND status = ?7", @@ -1847,7 +2221,7 @@ impl SqliteStore { params![ effect_id, encode_enum(EffectStatus::Uncertain)?, - error, + protect_text(&self.protection, error, "effects.error")?, now.to_rfc3339(), encode_enum(EffectStatus::Started)? ], @@ -1898,8 +2272,16 @@ impl SqliteStore { idempotency: decode_enum(&row.8, "effect.idempotency")?, idempotency_key: row.9, input_digest: row.10, - input: decode(&row.11, "effect.input")?, - expected_effect: row.12, + input: decode_protected( + &self.protection, + &row.11, + "effects.input_json", + )?, + expected_effect: expose_text( + &self.protection, + &row.12, + "effects.expected_effect", + )?, trace_id: row.13, }, status: decode_enum(&row.14, "effect.status")?, @@ -1907,8 +2289,20 @@ impl SqliteStore { requested_at: parse_time(&row.16, "effect.requested_at")?, started_at: row.17.map(|value| parse_time(&value, "effect.started_at")).transpose()?, completed_at: row.18.map(|value| parse_time(&value, "effect.completed_at")).transpose()?, - result: row.19.map(|value| decode(&value, "effect.result")).transpose()?, - error: row.20, + result: row + .19 + .map(|value| { + decode_protected( + &self.protection, + &value, + "effects.result_json", + ) + }) + .transpose()?, + error: row + .20 + .map(|value| expose_text(&self.protection, &value, "effects.error")) + .transpose()?, confirmed: row.21, }) }) @@ -1976,7 +2370,7 @@ impl SqliteStore { decode_reconciliation_row, ) .optional()? - .map(reconciliation_from_row) + .map(|row| reconciliation_from_row(row, &self.protection)) .transpose()?; if let Some(previous) = &previous { @@ -2061,11 +2455,39 @@ impl SqliteStore { source.0, encode_enum(request.status)?, request.actor, - request.reason, - encode(&request.evidence)?, - request.result.as_ref().map(encode).transpose()?, - request.result_schema.as_ref().map(encode).transpose()?, - encode(&request.authorization)?, + protect_text( + &self.protection, + &request.reason, + "effect_reconciliations.reason" + )?, + encode_protected( + &self.protection, + &request.evidence, + "effect_reconciliations.evidence_json" + )?, + request + .result + .as_ref() + .map(|value| encode_protected( + &self.protection, + value, + "effect_reconciliations.result_json" + )) + .transpose()?, + request + .result_schema + .as_ref() + .map(|value| encode_protected( + &self.protection, + value, + "effect_reconciliations.result_schema_json" + )) + .transpose()?, + encode_protected( + &self.protection, + &request.authorization, + "effect_reconciliations.authorization_json" + )?, request.compensation_effect_id, supersedes_id, request.trace_id, @@ -2091,6 +2513,7 @@ impl SqliteStore { &request.trace_id, &payload, now, + &self.protection, )?; append_trace_tx( &transaction, @@ -2107,6 +2530,7 @@ impl SqliteStore { "attributes": payload, }), now, + &self.protection, )?; transaction.commit()?; Ok(EffectReconciliationRecord { @@ -2174,7 +2598,7 @@ impl SqliteStore { decode_reconciliation_row, ) .optional()? - .map(reconciliation_from_row) + .map(|row| reconciliation_from_row(row, &self.protection)) .transpose() } @@ -2191,7 +2615,7 @@ impl SqliteStore { )?; statement .query_map([effect_id], decode_reconciliation_row)? - .map(|row| reconciliation_from_row(row?)) + .map(|row| reconciliation_from_row(row?, &self.protection)) .collect() } @@ -2208,7 +2632,7 @@ impl SqliteStore { )?; statement .query_map([run_id], decode_reconciliation_row)? - .map(|row| reconciliation_from_row(row?)) + .map(|row| reconciliation_from_row(row?, &self.protection)) .collect() } @@ -2234,7 +2658,29 @@ impl SqliteStore { let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; transaction.execute( "INSERT INTO approvals (approval_id, run_id, effect_id, task_id, agent, tool, capability, risk, redacted_input_json, expected_effect, reason, trace_id, status, requested_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'pending', ?13)", - params![request.approval_id, request.run_id, request.effect_id, request.task_id, request.agent, request.tool, request.capability, request.risk, encode(&request.redacted_input)?, request.expected_effect, request.reason, request.trace_id, request.requested_at.to_rfc3339()], + params![ + request.approval_id, + request.run_id, + request.effect_id, + request.task_id, + request.agent, + request.tool, + request.capability, + request.risk, + encode_protected( + &self.protection, + &request.redacted_input, + "approvals.redacted_input_json" + )?, + protect_text( + &self.protection, + &request.expected_effect, + "approvals.expected_effect" + )?, + protect_text(&self.protection, &request.reason, "approvals.reason")?, + request.trace_id, + request.requested_at.to_rfc3339() + ], )?; transaction.execute( "UPDATE effects SET status = ?2 WHERE effect_id = ?1", @@ -2273,7 +2719,17 @@ impl SqliteStore { }; transaction.execute( "UPDATE approvals SET status = ?2, resolved_at = ?3, resolved_by = ?4, resolution_reason = ?5 WHERE approval_id = ?1", - params![approval_id, status, now.to_rfc3339(), actor, reason], + params![ + approval_id, + status, + now.to_rfc3339(), + actor, + protect_text( + &self.protection, + reason, + "approvals.resolution_reason" + )? + ], )?; let effect_status = match resolution { ApprovalResolution::Approved => EffectStatus::Requested, @@ -2320,9 +2776,17 @@ impl SqliteStore { tool: row.4, capability: row.5, risk: row.6, - redacted_input: decode(&row.7, "approval.input")?, - expected_effect: row.8, - reason: row.9, + redacted_input: decode_protected( + &self.protection, + &row.7, + "approvals.redacted_input_json", + )?, + expected_effect: expose_text( + &self.protection, + &row.8, + "approvals.expected_effect", + )?, + reason: expose_text(&self.protection, &row.9, "approvals.reason")?, trace_id: row.10, requested_at: parse_time(&row.11, "approval.requested_at")?, }) @@ -2353,6 +2817,7 @@ impl SqliteStore { trace_id, &Value::Null, now, + &self.protection, )?; transaction.commit()?; Ok(()) @@ -2388,7 +2853,11 @@ impl SqliteStore { event_type: row.1, task_id: row.2, trace_id: row.3, - payload: decode(&row.4, "audit.payload")?, + payload: decode_protected( + &self.protection, + &row.4, + "audit_events.payload_json", + )?, created_at: parse_time(&row.5, "audit.created_at")?, }) }) @@ -2441,7 +2910,7 @@ impl SqliteStore { Ok(CheckpointRecord { sequence: row.0, format_version: row.1, - state: decode(&row.2, "checkpoint.state")?, + state: decode_protected(&self.protection, &row.2, "checkpoints.state_json")?, checksum: row.3, created_at: parse_time(&row.4, "checkpoint.created_at")?, }) @@ -2479,7 +2948,11 @@ impl SqliteStore { task_id: row.0, provider: row.1, format_version: row.2, - continuation: decode(&row.3, "provider_session.continuation")?, + continuation: decode_protected( + &self.protection, + &row.3, + "provider_sessions.continuation_json", + )?, updated_at: parse_time(&row.4, "provider_session.updated_at")?, }) }) @@ -2540,7 +3013,13 @@ impl SqliteStore { )?; connection.execute( "INSERT INTO trace_events (run_id, sequence, trace_id, event_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5)", - params![run_id, sequence, trace_id, encode(event)?, now.to_rfc3339()], + params![ + run_id, + sequence, + trace_id, + encode_protected(&self.protection, event, "trace_events.event_json")?, + now.to_rfc3339() + ], )?; Ok(()) } @@ -2564,7 +3043,7 @@ impl SqliteStore { Ok(TraceRecord { sequence: row.0, trace_id: row.1, - event: decode(&row.2, "trace.event")?, + event: decode_protected(&self.protection, &row.2, "trace_events.event_json")?, created_at: parse_time(&row.3, "trace.created_at")?, }) }) @@ -2581,7 +3060,13 @@ impl SqliteStore { ) -> Result<(), StoreError> { self.connection.lock().execute( "INSERT INTO long_term_memory (namespace, memory_key, value_json, expires_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT(namespace, memory_key) DO UPDATE SET value_json = excluded.value_json, expires_at = excluded.expires_at, updated_at = excluded.updated_at", - params![namespace, key, encode(value)?, expires_at.map(|value| value.to_rfc3339()), now.to_rfc3339()], + params![ + namespace, + key, + encode_protected(&self.protection, value, "long_term_memory.value_json")?, + expires_at.map(|value| value.to_rfc3339()), + now.to_rfc3339() + ], )?; Ok(()) } @@ -2596,7 +3081,17 @@ impl SqliteStore { ) -> Result<(), StoreError> { self.connection.lock().execute( "INSERT INTO provider_sessions (run_id, task_id, provider, format_version, continuation_json, updated_at) VALUES (?1, ?2, ?3, 1, ?4, ?5) ON CONFLICT(run_id, task_id) DO UPDATE SET provider = excluded.provider, format_version = excluded.format_version, continuation_json = excluded.continuation_json, updated_at = excluded.updated_at", - params![run_id, task_id, provider, encode(continuation)?, now.to_rfc3339()], + params![ + run_id, + task_id, + provider, + encode_protected( + &self.protection, + continuation, + "provider_sessions.continuation_json" + )?, + now.to_rfc3339() + ], )?; Ok(()) } @@ -2631,12 +3126,22 @@ impl SqliteStore { let (effect_status, output, error, confirmed, call_status) = match result { Ok(output) => ( EffectStatus::Succeeded, - Some(encode(output)?), + Some(encode_protected( + &self.protection, + output, + "effects.result_json", + )?), None, true, "succeeded", ), - Err(error) => (EffectStatus::Failed, None, Some(error), false, "failed"), + Err(error) => ( + EffectStatus::Failed, + None, + Some(protect_text(&self.protection, error, "effects.error")?), + false, + "failed", + ), }; let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; @@ -2672,7 +3177,13 @@ impl SqliteStore { let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; let effect_changed = transaction.execute( "UPDATE effects SET status = ?2, error = ?3, completed_at = ?4, confirmed = 0 WHERE effect_id = ?1 AND status = ?5", - params![effect_id, encode_enum(EffectStatus::Uncertain)?, error, now.to_rfc3339(), encode_enum(EffectStatus::Started)?], + params![ + effect_id, + encode_enum(EffectStatus::Uncertain)?, + protect_text(&self.protection, error, "effects.error")?, + now.to_rfc3339(), + encode_enum(EffectStatus::Started)? + ], )?; if effect_changed != 1 { return Err(StoreError::EffectNotFound(effect_id.to_owned())); @@ -2702,7 +3213,7 @@ impl SqliteStore { |row| row.get(0), ).optional()?; value - .map(|value| decode(&value, "long_term_memory.value")) + .map(|value| decode_protected(&self.protection, &value, "long_term_memory.value_json")) .transpose() } @@ -3093,6 +3604,136 @@ fn configure(connection: &Connection) -> Result<(), StoreError> { Ok(()) } +fn load_state_protection( + connection: &Connection, + resolver: &dyn StateKeyResolver, +) -> Result { + let config = connection + .query_row( + "SELECT format_version, key_id, key_reference, key_check, maintenance FROM state_encryption WHERE singleton = 1", + [], + |row| { + Ok(( + row.get::<_, u32>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, bool>(4)?, + )) + }, + ) + .optional()?; + let Some((format_version, key_id, key_reference, key_check, maintenance)) = config else { + return Ok(StateProtection::Plaintext); + }; + if maintenance { + return Err(StoreError::Encryption( + "state-encryption maintenance transaction is incomplete".to_owned(), + )); + } + if format_version != encryption::ENCRYPTION_FORMAT_VERSION { + return Err(StoreError::Encryption(format!( + "unsupported state-encryption configuration version {format_version}" + ))); + } + let codec = EncryptionCodec::resolve(&key_id, &key_reference, resolver)?; + codec.verify_key_check(&key_check)?; + Ok(StateProtection::Encrypted(codec)) +} + +fn encryption_inventory( + connection: &Connection, + protection: &StateProtection, +) -> Result { + let config = connection + .query_row( + "SELECT key_id, key_reference FROM state_encryption WHERE singleton = 1", + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + let mut protected_values = 0_u64; + let mut encrypted_values = 0_u64; + let mut plaintext_values = 0_u64; + let mut invalid_envelopes = 0_u64; + for column in SENSITIVE_COLUMNS { + let context = column.context(); + let sql = format!( + "SELECT {} FROM {} WHERE {} IS NOT NULL", + column.column, column.table, column.column + ); + let mut statement = connection.prepare(&sql)?; + let values = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + for value in values { + protected_values = protected_values.saturating_add(1); + if is_encrypted_value(&value) { + encrypted_values = encrypted_values.saturating_add(1); + let valid = if protection.is_enabled() { + protection.expose(&value, &context).map(|_| ()) + } else { + validate_envelope(&value, &context) + }; + if valid.is_err() { + invalid_envelopes = invalid_envelopes.saturating_add(1); + } + } else { + plaintext_values = plaintext_values.saturating_add(1); + } + } + } + Ok(EncryptionInventory { + enabled: config.is_some(), + key_id: config.as_ref().map(|value| value.0.clone()), + key_reference: config.map(|value| value.1), + protected_values, + encrypted_values, + plaintext_values, + invalid_envelopes, + }) +} + +fn rewrite_sensitive_values( + transaction: &Transaction<'_>, + current: &StateProtection, + next: &StateProtection, +) -> Result<(), StoreError> { + for column in SENSITIVE_COLUMNS { + let context = column.context(); + let select = format!( + "SELECT rowid, {} FROM {} WHERE {} IS NOT NULL", + column.column, column.table, column.column + ); + let values = { + let mut statement = transaction.prepare(&select)?; + statement + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()? + }; + for (row_id, stored) in values { + let plaintext = current.expose(&stored, &context)?; + let protected = next.protect(&plaintext, &context)?; + if column.table == "checkpoints" && column.column == "state_json" { + let checksum = hex::encode(Sha256::digest(protected.as_bytes())); + transaction.execute( + "UPDATE checkpoints SET state_json = ?1, checksum = ?2 WHERE rowid = ?3", + params![protected, checksum, row_id], + )?; + } else { + let update = format!( + "UPDATE {} SET {} = ?1 WHERE rowid = ?2", + column.table, column.column + ); + transaction.execute(&update, params![protected, row_id])?; + } + } + } + Ok(()) +} + fn migrate(connection: &mut Connection) -> Result<(), StoreError> { let current: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; if current > DATABASE_SCHEMA_VERSION { @@ -3111,6 +3752,7 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { (7_u32, MIGRATION_7), (8_u32, MIGRATION_8), (9_u32, MIGRATION_9), + (10_u32, MIGRATION_10), ]; for (version, sql) in migrations .into_iter() @@ -3121,6 +3763,46 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { transaction.pragma_update(None, "user_version", version)?; transaction.commit()?; } + install_encryption_triggers(connection)?; + Ok(()) +} + +fn install_encryption_triggers(connection: &Connection) -> Result<(), StoreError> { + for column in SENSITIVE_COLUMNS { + let insert_name = format!( + "enforce_encryption_{}_{}_insert", + column.table, column.column + ); + let update_name = format!( + "enforce_encryption_{}_{}_update", + column.table, column.column + ); + let expected_prefix = format!( + "'{ENVELOPE_PREFIX}' || (SELECT key_id FROM state_encryption WHERE singleton = 1) || ':'" + ); + let condition = format!( + "(SELECT COUNT(*) FROM state_encryption WHERE singleton = 1 AND maintenance = 0) = 1 + AND NEW.{column} IS NOT NULL + AND substr(NEW.{column}, 1, length({expected_prefix})) != {expected_prefix}", + column = column.column + ); + connection.execute_batch(&format!( + "CREATE TRIGGER IF NOT EXISTS {insert_name} + BEFORE INSERT ON {table} + WHEN {condition} + BEGIN + SELECT RAISE(ABORT, 'protected field requires the current state-encryption key'); + END; + CREATE TRIGGER IF NOT EXISTS {update_name} + BEFORE UPDATE OF {column} ON {table} + WHEN {condition} + BEGIN + SELECT RAISE(ABORT, 'protected field requires the current state-encryption key'); + END;", + table = column.table, + column = column.column, + ))?; + } Ok(()) } @@ -3128,6 +3810,7 @@ fn checkpoint_tx( transaction: &Transaction<'_>, run_id: &str, now: DateTime, + protection: &SharedStateProtection, ) -> Result<(), StoreError> { let run_state: (String, String, Option, bool) = transaction.query_row( "SELECT state, working_memory_json, output_json, cancellation_requested FROM runs WHERE run_id = ?1", @@ -3137,26 +3820,52 @@ fn checkpoint_tx( let mut statement = transaction.prepare( "SELECT task_id, state, attempt, output_json, error FROM task_states WHERE run_id = ?1 ORDER BY position", )?; - let tasks: Vec = statement + let task_rows = statement .query_map([run_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, u16>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + )) + })? + .collect::, _>>()?; + let tasks = task_rows + .into_iter() + .map(|row| { Ok(serde_json::json!({ - "taskId": row.get::<_, String>(0)?, - "state": row.get::<_, String>(1)?, - "attempt": row.get::<_, u16>(2)?, - "output": row.get::<_, Option>(3)?.and_then(|raw| serde_json::from_str::(&raw).ok()), - "error": row.get::<_, Option>(4)?, + "taskId": row.0, + "state": row.1, + "attempt": row.2, + "output": row.3 + .map(|raw| decode_protected::( + protection, + &raw, + "task_states.output_json" + )) + .transpose()?, + "error": row.4 + .map(|raw| expose_text(protection, &raw, "task_states.error")) + .transpose()?, })) - })? - .collect::>()?; + }) + .collect::, StoreError>>()?; let state = serde_json::json!({ "runId": run_id, "state": run_state.0, - "workingMemory": decode::(&run_state.1, "working_memory")?, - "output": run_state.2.map(|raw| decode::(&raw, "output")).transpose()?, + "workingMemory": decode_protected::( + protection, + &run_state.1, + "runs.working_memory_json" + )?, + "output": run_state.2 + .map(|raw| decode_protected::(protection, &raw, "runs.output_json")) + .transpose()?, "cancellationRequested": run_state.3, "tasks": tasks, }); - let state_json = encode(&state)?; + let state_json = encode_protected(protection, &state, "checkpoints.state_json")?; let checksum = hex::encode(Sha256::digest(state_json.as_bytes())); let sequence: i64 = transaction.query_row( "SELECT COALESCE(MAX(sequence), 0) + 1 FROM checkpoints WHERE run_id = ?1", @@ -3170,6 +3879,7 @@ fn checkpoint_tx( Ok(()) } +#[allow(clippy::too_many_arguments)] fn append_audit_tx( transaction: &Transaction<'_>, run_id: &str, @@ -3178,6 +3888,7 @@ fn append_audit_tx( trace_id: &str, payload: &Value, now: DateTime, + protection: &SharedStateProtection, ) -> Result<(), StoreError> { let sequence: i64 = transaction.query_row( "SELECT COALESCE(MAX(sequence), 0) + 1 FROM audit_events WHERE run_id = ?1", @@ -3186,7 +3897,16 @@ fn append_audit_tx( )?; transaction.execute( "INSERT INTO audit_events (run_id, sequence, event_version, event_type, task_id, trace_id, payload_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - params![run_id, sequence, AUDIT_EVENT_VERSION, event_type, task_id, trace_id, encode(payload)?, now.to_rfc3339()], + params![ + run_id, + sequence, + AUDIT_EVENT_VERSION, + event_type, + task_id, + trace_id, + encode_protected(protection, payload, "audit_events.payload_json")?, + now.to_rfc3339() + ], )?; Ok(()) } @@ -3197,6 +3917,7 @@ fn append_trace_tx( trace_id: &str, event: &Value, now: DateTime, + protection: &SharedStateProtection, ) -> Result<(), StoreError> { let sequence: i64 = transaction.query_row( "SELECT COALESCE(MAX(sequence), 0) + 1 FROM trace_events WHERE run_id = ?1", @@ -3209,7 +3930,7 @@ fn append_trace_tx( run_id, sequence, trace_id, - encode(event)?, + encode_protected(protection, event, "trace_events.event_json")?, now.to_rfc3339() ], )?; @@ -3256,6 +3977,7 @@ fn decode_reconciliation_row(row: &rusqlite::Row<'_>) -> rusqlite::Result Result { if row.3 != 1 { return Err(StoreError::Incompatible(format!( @@ -3270,17 +3992,27 @@ fn reconciliation_from_row( format_version: row.3, status: decode_enum(&row.4, "effect_reconciliation.status")?, actor: row.5, - reason: row.6, - evidence: decode(&row.7, "effect_reconciliation.evidence")?, + reason: expose_text(protection, &row.6, "effect_reconciliations.reason")?, + evidence: decode_protected(protection, &row.7, "effect_reconciliations.evidence_json")?, result: row .8 - .map(|value| decode(&value, "effect_reconciliation.result")) + .map(|value| decode_protected(protection, &value, "effect_reconciliations.result_json")) .transpose()?, result_schema: row .9 - .map(|value| decode(&value, "effect_reconciliation.result_schema")) + .map(|value| { + decode_protected( + protection, + &value, + "effect_reconciliations.result_schema_json", + ) + }) .transpose()?, - authorization: decode(&row.10, "effect_reconciliation.authorization")?, + authorization: decode_protected( + protection, + &row.10, + "effect_reconciliations.authorization_json", + )?, compensation_effect_id: row.11, supersedes_id: row.12, trace_id: row.13, @@ -3452,6 +4184,39 @@ fn encode(value: &T) -> Result { serde_json::to_string(value).map_err(StoreError::from) } +fn encode_protected( + protection: &SharedStateProtection, + value: &T, + context: &str, +) -> Result { + protection.read().protect(&encode(value)?, context) +} + +fn decode_protected( + protection: &SharedStateProtection, + value: &str, + context: &str, +) -> Result { + let plaintext = protection.read().expose(value, context)?; + decode(&plaintext, context) +} + +fn protect_text( + protection: &SharedStateProtection, + value: &str, + context: &str, +) -> Result { + protection.read().protect(value, context) +} + +fn expose_text( + protection: &SharedStateProtection, + value: &str, + context: &str, +) -> Result { + protection.read().expose(value, context) +} + fn encode_enum(value: T) -> Result { let value = serde_json::to_value(value)?; value @@ -3481,6 +4246,39 @@ mod tests { use agentctl_core::dsl::{API_VERSION, EffectClass, Idempotency, Risk, parse_workflow}; use agentctl_core::effect::EffectRequest; use tempfile::tempdir; + use zeroize::Zeroizing; + + #[derive(Default)] + struct FixedKeyResolver { + keys: BTreeMap>, + } + + impl FixedKeyResolver { + fn with(reference: &str, byte: u8) -> Self { + Self { + keys: BTreeMap::from([(reference.to_owned(), vec![byte; 32])]), + } + } + + fn and(mut self, reference: &str, byte: u8) -> Self { + self.keys.insert(reference.to_owned(), vec![byte; 32]); + self + } + } + + impl StateKeyResolver for FixedKeyResolver { + fn resolve(&self, reference: &str) -> Result>, StoreError> { + self.keys + .get(reference) + .cloned() + .map(Zeroizing::new) + .ok_or_else(|| { + StoreError::Encryption(format!( + "fixed key reference `{reference}` is unavailable" + )) + }) + } + } fn fixture() -> (Value, CompiledPlan) { let source = r#" @@ -3582,6 +4380,7 @@ spec: MIGRATION_6, MIGRATION_7, MIGRATION_8, + MIGRATION_9, ] .into_iter() .enumerate() @@ -3616,6 +4415,283 @@ spec: } } + #[test] + fn authenticated_envelope_binds_key_and_field_context() { + let codec = EncryptionCodec::from_bytes("key-2026", vec![7; 32]).expect("encryption codec"); + let protected = codec + .encrypt(r#"{"secret":"value"}"#, "runs.inputs_json") + .expect("encrypt"); + assert!(protected.starts_with("agentctl.encrypted.v1:key-2026:")); + assert!(!protected.contains(r#""secret":"value""#)); + assert_eq!( + codec + .decrypt(&protected, "runs.inputs_json") + .expect("decrypt"), + r#"{"secret":"value"}"# + ); + assert!(codec.decrypt(&protected, "runs.output_json").is_err()); + let mut tampered = protected; + tampered.push('x'); + assert!(codec.decrypt(&tampered, "runs.inputs_json").is_err()); + } + + #[test] + fn encryption_inventory_migration_rotation_and_fail_closed_reads_are_transactional() { + let directory = tempdir().expect("temp dir"); + let path = directory.path().join("runtime.db"); + let resolver = Arc::new( + FixedKeyResolver::with("AGENTCTL_TEST_OLD_KEY", 1).and("AGENTCTL_TEST_NEW_KEY", 2), + ); + let store = + SqliteStore::open_with_key_resolver(&path, resolver.clone()).expect("open store"); + let marker = "protected-marker-value"; + let (mut workflow, plan) = fixture(); + workflow["testSensitiveValue"] = Value::String(marker.to_owned()); + store + .create_run( + "encrypted-run", + API_VERSION, + &workflow, + &plan, + &serde_json::json!({"secret": marker}), + &serde_json::json!({"working": marker}), + RunMode::Execute, + None, + directory.path(), + Utc::now(), + "trace-encryption", + ) + .expect("create run"); + let effect = EffectRequest::new( + "encrypted-run", + "one", + 1, + 1, + "test.sensitive", + EffectClass::Observe, + Risk::Low, + Idempotency::Idempotent, + serde_json::json!({"secret": marker}), + marker, + "trace-encryption", + ); + store + .record_effect_request(&effect, Utc::now()) + .expect("effect request"); + store + .put_provider_session( + "encrypted-run", + "one", + "fake", + &serde_json::json!({"opaque": marker}), + Utc::now(), + ) + .expect("provider session"); + store + .put_long_term_memory( + "test", + "secret", + &serde_json::json!({"value": marker}), + None, + Utc::now(), + ) + .expect("memory"); + store + .record_trace_event( + "encrypted-run", + "trace-encryption", + &serde_json::json!({"detail": marker}), + Utc::now(), + ) + .expect("trace"); + + let before = store.encryption_inventory().expect("inventory"); + assert!(!before.enabled); + assert!(before.plaintext_values > 0); + let dry_run = store + .enable_encryption("keyXv1", "AGENTCTL_TEST_OLD_KEY", true, Utc::now()) + .expect("dry run"); + assert!(dry_run.dry_run); + assert_eq!(dry_run.values_rewritten, 0); + assert!(!store.encryption_inventory().expect("inventory").enabled); + + let enabled = store + .enable_encryption("keyXv1", "AGENTCTL_TEST_OLD_KEY", false, Utc::now()) + .expect("enable encryption"); + assert!(enabled.values_rewritten > 0); + let inventory = store.encryption_inventory().expect("encrypted inventory"); + assert!(inventory.enabled); + assert_eq!(inventory.key_id.as_deref(), Some("keyXv1")); + assert_eq!(inventory.plaintext_values, 0); + assert_eq!(inventory.invalid_envelopes, 0); + assert_eq!(inventory.protected_values, inventory.encrypted_values); + assert!( + !serde_json::to_string(&inventory) + .expect("inventory json") + .contains(marker) + ); + assert_eq!( + store.load_run("encrypted-run").expect("run").inputs["secret"], + marker + ); + assert_eq!( + store.load_effect(&effect.id).expect("effect").request.input["secret"], + marker + ); + assert_eq!( + store + .get_long_term_memory("test", "secret", Utc::now()) + .expect("memory") + .expect("present")["value"], + marker + ); + assert!( + !store + .checkpoints("encrypted-run") + .expect("checkpoints") + .is_empty() + ); + + let stale_ciphertext = { + let connection = Connection::open(&path).expect("raw connection"); + for column in SENSITIVE_COLUMNS { + let sql = format!( + "SELECT {} FROM {} WHERE {} IS NOT NULL", + column.column, column.table, column.column + ); + let mut statement = connection.prepare(&sql).expect("prepare"); + let values = statement + .query_map([], |row| row.get::<_, String>(0)) + .expect("query") + .collect::, _>>() + .expect("values"); + for value in values { + assert!(is_encrypted_value(&value), "{}", column.context()); + assert!(!value.contains(marker), "{}", column.context()); + } + } + assert!( + connection + .execute( + "UPDATE runs SET inputs_json = 'plaintext' WHERE run_id = 'encrypted-run'", + [], + ) + .is_err() + ); + connection + .query_row( + "SELECT inputs_json FROM runs WHERE run_id = 'encrypted-run'", + [], + |row| row.get::<_, String>(0), + ) + .expect("stale ciphertext") + }; + + let wrong = Arc::new( + FixedKeyResolver::with("AGENTCTL_TEST_OLD_KEY", 9).and("AGENTCTL_TEST_NEW_KEY", 2), + ); + assert!(SqliteStore::open_with_key_resolver(&path, wrong).is_err()); + + store + .connection() + .execute_batch( + "CREATE TRIGGER fail_test_rotation + BEFORE UPDATE OF inputs_json ON runs + WHEN (SELECT maintenance FROM state_encryption WHERE singleton = 1) = 1 + BEGIN + SELECT RAISE(ABORT, 'injected rotation failure'); + END;", + ) + .expect("failure trigger"); + assert!( + store + .rotate_encryption_key("key_v1", "AGENTCTL_TEST_NEW_KEY", false, Utc::now(),) + .is_err() + ); + assert_eq!( + store + .encryption_inventory() + .expect("post-rollback inventory") + .key_id + .as_deref(), + Some("keyXv1") + ); + assert_eq!( + store + .load_run("encrypted-run") + .expect("rollback run") + .inputs["secret"], + marker + ); + store + .connection() + .execute_batch("DROP TRIGGER fail_test_rotation") + .expect("drop trigger"); + + let rotation_plan = store + .rotate_encryption_key("key_v1", "AGENTCTL_TEST_NEW_KEY", true, Utc::now()) + .expect("rotation plan"); + assert!(rotation_plan.dry_run); + let rotated = store + .rotate_encryption_key("key_v1", "AGENTCTL_TEST_NEW_KEY", false, Utc::now()) + .expect("rotate"); + assert!(rotated.values_rewritten > 0); + assert_eq!( + store + .encryption_inventory() + .expect("rotated inventory") + .key_id + .as_deref(), + Some("key_v1") + ); + assert!( + Connection::open(&path) + .expect("raw connection") + .execute( + "UPDATE runs SET inputs_json = ?1 WHERE run_id = 'encrypted-run'", + [stale_ciphertext], + ) + .is_err() + ); + let new_only = Arc::new(FixedKeyResolver::with("AGENTCTL_TEST_NEW_KEY", 2)); + let reopened = + SqliteStore::open_with_key_resolver(&path, new_only).expect("open with new key"); + assert_eq!( + reopened + .load_run("encrypted-run") + .expect("rotated run") + .inputs["secret"], + marker + ); + assert!( + !reopened + .checkpoints("encrypted-run") + .expect("rotated checkpoints") + .is_empty() + ); + + let old_for_new = Arc::new(FixedKeyResolver::with("AGENTCTL_TEST_NEW_KEY", 1)); + assert!(SqliteStore::open_with_key_resolver(&path, old_for_new).is_err()); + { + let connection = Connection::open(&path).expect("raw connection"); + let mut stored: String = connection + .query_row( + "SELECT inputs_json FROM runs WHERE run_id = 'encrypted-run'", + [], + |row| row.get(0), + ) + .expect("stored input"); + stored.push('x'); + connection + .execute( + "UPDATE runs SET inputs_json = ?1 WHERE run_id = 'encrypted-run'", + [stored], + ) + .expect("tamper"); + } + assert!(reopened.load_run("encrypted-run").is_err()); + } + #[test] fn unknown_future_schema_fails_explicitly() { let directory = tempdir().expect("temp dir"); diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index f63caf4..94a0b6c 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -15,6 +15,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an - Non-interactive approvals durably pause, signals cancel safely, JSON errors include available run/trace correlation, and SQLite uses WAL plus a bounded lock wait. - The packaged CLI, clean-directory quickstart, cron-like empty environment, and non-root/read-only OCI contract have executable acceptance coverage. - Successful bounded file outputs are atomically ingested into a local immutable content-addressed store with durable references, verification/export commands, lease-safe reachability GC, interrupted-GC recovery, and local/OCI acceptance coverage. +- Identified confidential JSON and text fields can be transactionally migrated to versioned AES-256-GCM envelopes, rotated through environment key references, inventoried without content disclosure, and fail closed on missing/wrong keys, tampering, plaintext writes, or stale-key writes. - Shell execution and acceptance/container helpers use bounded concurrent capture. Output overflow terminates/reaps the child with a structured secret-safe error; timeouts and cancellation retain durable uncertain-effect semantics. - Hosted workflows use least privilege, full-SHA action pins with version comments, complete-history/tree Gitleaks, deterministic fake-secret detection, dependency/image scans, and required production/image CycloneDX artifacts with digests. @@ -28,7 +29,7 @@ These are useful extensions but are not required by the product thesis. They nee - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; - pack dependency resolution, pack lockfiles, remote fetching, publisher signatures, and a versioned plugin ABI; - vector memory; -- encrypted application-level persistence and external secret-manager adapters; +- external secret-manager adapters beyond environment, mounted-file, and policy-gated process references; - reliable monetary cost enforcement when providers expose sufficient authoritative metadata. ## Explicit non-goals @@ -43,6 +44,7 @@ These are useful extensions but are not required by the product thesis. They nee - The document API is `v1alpha1`; pin the binary/image version and validate before upgrading. - Scheduling is sequential (`maxConcurrency: 1`). Separate runs may overlap safely in SQLite, but they can still target the same external resource. Use the external scheduler's overlap controls (`flock`, systemd unit serialization, or Kubernetes `concurrencyPolicy: Forbid`) when effects must not overlap. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. +- State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. - At-most-once model/remote calls can become uncertain in the dispatch/acknowledgement window. Inspect and reconcile externally; use `fork` only when fresh effects are knowingly acceptable. - Successful tasks from databases created before schema 5 require explicit `runs analyze`/`runs upgrade`. Only provable metadata is imported; unprovable boundaries are returned as conservative safe repair roots. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index b830b19..6b7d623 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -10,6 +10,7 @@ - Tool input and output JSON Schemas are enforced. Models, MCP annotations, A2A cards, remote schemas, and results cannot grant capabilities. - Requests are ledgered before effects. Global denial or approval cannot be weakened by a tool contract. Approval is durable; non-interactive mode pauses with exit `3` or uses an explicitly stricter deny/fail mode, never a prompt or implicit approval. - SQLite uses foreign keys, WAL/busy timeout, version checks, checksummed checkpoints, and mode `0600` on Unix. +- Optional application-level state encryption uses versioned AES-256-GCM envelopes with per-value random nonces, field-bound authenticated data, key IDs, environment references, transactional migration/rotation, and database triggers that reject plaintext or stale-key writes after enablement. Missing, wrong, unsupported, or tampered keys/envelopes fail closed. - Repair never mutates a terminal source. Reuse requires versioned definition/input/contract/output/state metadata and verified content-addressed artifact sizes and SHA-256 digests. Artifact ingestion uses atomic no-clobber writes, immutable blobs, bounded leases, and a cross-process GC lock. Repair creation and reused-task/reference materialization are one SQLite transaction. - A recorded replay cannot be a repair source because it has no direct effect ledger. A materialized reused/recorded task cannot be selected for restart without returning to direct effect history. Repaired agents start fresh provider sessions. - Packs require a supported manifest/version and can be checked against SHA-256 integrity. @@ -17,12 +18,12 @@ ## Limitations -Path and executable allowlists are not a sandbox. A permitted program can access anything the operating-system identity can access. Host allowlists do not defend against every DNS rebinding, proxy, local-service, or compromised endpoint scenario; use network isolation for hostile workflows. SHA-256 integrity establishes sameness, not author identity. SQLite protects local correctness but is not encrypted and is not a secret store. +Path and executable allowlists are not a sandbox. A permitted program can access anything the operating-system identity can access. Host allowlists do not defend against every DNS rebinding, proxy, local-service, or compromised endpoint scenario; use network isolation for hostile workflows. SHA-256 integrity establishes sameness, not author identity. State encryption is application-level selected-field protection, not full-database encryption, access control, or a secret store. Prompts, file content, model output, remote artifacts, and tool output may be confidential or malicious. Treat them as data, validate before mutation, minimize trace export, and isolate untrusted automation. Workflow, input, pack, direct-read, existing-write-target, and instruction files are capped at 1 MiB. Approval is a decision point, not proof that an operation is safe. At-most-once recovery may leave an uncertain external outcome for human reconciliation. MCP reconnection and A2A resubmission are intentionally not automatic. Streaming is bounded but completed results, not token deltas, enter workflow state. Windows cannot express Unix database mode bits; rely on the user profile ACL and CI tests. -SQLite and sibling artifact-root access are the repair authorization boundary. There is no tenant identity or row-level authorization. Artifact bytes are not encrypted; an identity that can modify the state directory can corrupt or replace local history, although digest verification prevents silent reuse of changed bytes. +SQLite and sibling artifact-root access are the repair authorization boundary. There is no tenant identity or row-level authorization. Run IDs, task/effect identity, status, timing, digests, paths, sizes, schema metadata, and key references remain visible. Artifact blob bytes are not encrypted. An identity that can modify the state directory can corrupt or replace local history, although envelope authentication and artifact digest verification prevent silent use of changed protected content. Report vulnerabilities privately to the repository maintainer. Do not include credentials, database contents, or production prompts in a report. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 2aa43a9..8498499 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -11,6 +11,7 @@ The local operator and reviewed binary are trusted. Workflow authors are only as | Malicious YAML/template causes code execution or resource exhaustion | strict fields, constrained paths/equality, 1 MiB bound, fuzzing | deeply nested valid data remains bounded mainly by parser behavior | | Path traversal or symlink escape | canonical roots and focused tests | TOCTOU is possible if another process swaps paths; isolate hostile workspaces | | Secret exfiltration through CLI/log/database/trace | env references, no key flags, allowlists, redaction, secret scan | authorized tools can deliberately transmit permitted data | +| SQLite disclosure reveals confidential run content | optional AES-256-GCM field envelopes, external key reference, authenticated context, fail-closed triggers, transactional rotation | metadata and artifact bytes remain visible; unencrypted and pre-migration backups remain sensitive | | Command injection | direct argv, no shell, cleared env, executable allowlist | an allowed executable may interpret malicious arguments | | SSRF/redirect bypass | URL parse, host allowlist, disabled redirects, tests | DNS/proxy behavior needs external network containment for hostile inputs | | Prompt injection grants tool authority | policy outside model, visible tool set, schema validation, approvals | an operator may approve deceptive content | @@ -27,6 +28,6 @@ The local operator and reviewed binary are trusted. Workflow authors are only as | Corrupt or future state misexecutes | schema/version/checksum/deserialization failures | SQLite file deletion or rollback by an attacker is not prevented | | Dependency compromise | locked registry-only deps, cargo-deny, license/source checks | registry compromise and zero-days remain possible | -No unresolved critical or high-severity defect is knowingly accepted for the implemented boundary. Deferred sandboxing, signature verification, distributed concurrency, and encrypted storage are explicit product limitations, not implied controls. +No unresolved critical or high-severity defect is knowingly accepted for the implemented boundary. Process sandboxing, signature verification, and distributed concurrency are not implied controls. State encryption protects its documented columns only and is not described as full-database encryption. Run access control is the database file and operating-system identity. `agentctl` has no multi-tenant authorization layer; do not let an untrusted principal select another tenant's source run from a shared database. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index faeab91..1637799 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -72,6 +72,7 @@ cargo xtask acceptance-container | Legacy upgrades | all retained schema fixtures, dry-run, rollback, import, boundary, repair/replay tests | migration verification command added; full composite rerun pending | verified | | Reconciliation | immutable transition matrix, schema/tool/hook/policy, repair and resume tests | full composite rerun pending | verified | | Terminal retry | runtime/store identity, roots, acknowledgements, reconciliation, lineage, source immutability, and replay tests passed | packaged CLI scenario 30 and the 12-stage verification gate passed | verified | +| Sensitive-state encryption | authenticated context, wrong-key, tamper, inventory, stale-writer trigger, rollback, rotation, checkpoint, and retained-schema tests passed | packaged CLI scenario 31 and the 12-stage verification gate passed | verified | | Parallel/dynamic workflows | pending | pending | open | | Conditions/loops/sub-workflows | pending | pending | open | | Compensation/handoffs/streaming | pending | pending | open | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index b84fdcd..39efe36 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -38,7 +38,7 @@ complete, every entry must have exactly one final disposition: | MIG-001 | Legacy selective repair | verified | implemented | | EFX-001 | Effect reconciliation | verified | implemented | | RET-001 | Terminal-run retry | verified | implemented | -| ENC-001 | Sensitive-state encryption | open | implemented | +| ENC-001 | Sensitive-state encryption | verified | implemented | | SEC-001 | Secret providers | open | implemented | | NET-001 | Network policy | open | implemented | | ISO-001 | Process isolation | open | redesigned | @@ -178,8 +178,8 @@ complete, every entry must have exactly one final disposition: ### ENC-001: Envelope encryption for sensitive persisted fields -- Current behavior: SQLite may contain prompts, inputs, outputs, tool data, and - provider continuations in plaintext. +- Current behavior: identified confidential fields can be inventoried and + transactionally migrated to authenticated envelopes, then fail closed. - User impact: filesystem disclosure reveals confidential workflow history. - Security or durability impact: SQLite permissions are not confidentiality at rest. @@ -197,7 +197,11 @@ complete, every entry must have exactly one final disposition: - Examples: encrypted state with redacted inspection. - Live evidence: no provider call required. - Documentation: protected fields, key lifecycle, and residual metadata. -- Final disposition: pending implementation evidence. +- Final disposition: implemented and verified by authenticated-context + roundtrips, missing/wrong-key and tamper failure, dry-run inventory, + protected-column scans, trigger-enforced no-fallback writes, injected + rotation rollback, full atomic rotation, schema-10 fixture migration, normal + read compatibility, checkpoint integrity, and packaged CLI replay. ## Workflow language and runtime diff --git a/docs/generated/CLI.md b/docs/generated/CLI.md index 3e4aff7..cc11b03 100644 --- a/docs/generated/CLI.md +++ b/docs/generated/CLI.md @@ -689,6 +689,7 @@ Usage: agentctl db [OPTIONS] Commands: stats migrate + encryption Options: --db [default: .agentctl/runtime.db] @@ -698,6 +699,71 @@ Options: -h, --help Print help ``` +## `agentctl db encryption` + +```text +Usage: agentctl db encryption [OPTIONS] + +Commands: + inventory Inventory protected fields without exposing their values + enable Transactionally encrypt every identified sensitive field + rotate Transactionally decrypt and re-encrypt every protected field with a new key + +Options: + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl db encryption inventory` + +```text +Inventory protected fields without exposing their values + +Usage: agentctl db encryption inventory [OPTIONS] + +Options: + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --verbose + -h, --help Print help +``` + +## `agentctl db encryption enable` + +```text +Transactionally encrypt every identified sensitive field + +Usage: agentctl db encryption enable [OPTIONS] --key-id --key-env + +Options: + --key-id + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --key-env Environment variable containing a base64-encoded 32-byte key + --dry-run + --verbose + -h, --help Print help +``` + +## `agentctl db encryption rotate` + +```text +Transactionally decrypt and re-encrypt every protected field with a new key + +Usage: agentctl db encryption rotate [OPTIONS] --key-id --key-env + +Options: + --key-id + --output [default: human] [possible values: human, json] + --color [default: auto] [possible values: auto, always, never] + --key-env Environment variable containing a base64-encoded 32-byte key + --dry-run + --verbose + -h, --help Print help +``` + ## `agentctl memory` ```text diff --git a/docs/guides/SENSITIVE_STATE_ENCRYPTION.md b/docs/guides/SENSITIVE_STATE_ENCRYPTION.md new file mode 100644 index 0000000..f2b3cc8 --- /dev/null +++ b/docs/guides/SENSITIVE_STATE_ENCRYPTION.md @@ -0,0 +1,60 @@ +# Sensitive-state encryption + +`agentctl` can protect identified confidential SQLite fields with versioned AES-256-GCM envelopes. Encryption is explicit, application-level selected-field protection. It is not full-database encryption and does not encrypt artifact blob bytes or operational metadata. + +## Prepare the key reference + +Supply a base64-encoded 32-byte key through an environment variable. The CLI receives only the environment-variable name: + +```console +export AGENTCTL_STATE_KEY="$(openssl rand -base64 32)" +agentctl db --db .agentctl/runtime.db encryption inventory \ + --output json --color never +``` + +Protect the key with the platform's normal secret injection and backup controls. Do not put its value in YAML, CLI arguments, logs, shell history, runtime inputs, or repository files. The database stores the key ID and reference name, never the value. + +## Inventory and enable + +Preview the migration: + +```console +agentctl db --db .agentctl/runtime.db encryption enable \ + --key-id production-2026-01 \ + --key-env AGENTCTL_STATE_KEY \ + --dry-run \ + --output json \ + --color never +``` + +The report contains counts and key metadata, not protected values. Back up the SQLite database and WAL state, then run the same command without `--dry-run`. + +Enablement rewrites every non-null protected field inside one immediate SQLite transaction and updates checkpoint checksums. After commit, database triggers reject plaintext and envelopes from stale key IDs. Opening the database requires the referenced key and authenticates every protected value. There is no plaintext fallback. + +Protected fields include workflow definitions and plans, inputs, working memory, run/task output and errors, state deltas and reuse decisions, effect input/results/errors, approval content, checkpoints, audit and trace payloads, provider continuations, reconciliation evidence/results, run-upgrade records, and long-term-memory values. + +## Rotate + +Keep the current key reference available while planning and performing rotation: + +```console +export AGENTCTL_STATE_KEY_NEXT="$(openssl rand -base64 32)" +agentctl db --db .agentctl/runtime.db encryption rotate \ + --key-id production-2026-07 \ + --key-env AGENTCTL_STATE_KEY_NEXT \ + --dry-run + +agentctl db --db .agentctl/runtime.db encryption rotate \ + --key-id production-2026-07 \ + --key-env AGENTCTL_STATE_KEY_NEXT +``` + +Rotation decrypts with the current key and re-encrypts every protected value with fresh nonces and the new key in one transaction. Any authentication, write, or injected storage failure rolls the entire rotation back. After success, only the new reference is required. + +## Backup and restore + +Back up SQLite, its WAL state, and the sibling artifact root as one consistency set. Preserve the current key outside that backup; without it, protected content is intentionally unrecoverable. Retire plaintext pre-migration backups and old snapshots under the same confidentiality policy as their original workflow content. + +Visible residual metadata includes run/task/effect IDs, state and mode, timestamps, effect class and operation, digests, schema versions, artifact paths/names/media types/sizes/digests, and the encryption key ID/reference. Artifact blob bytes remain governed by filesystem or volume encryption. + +If the key is missing, wrong, or malformed, or an envelope was changed, every normal open fails with persistence exit code `5`. Restore the correct key or an internally consistent database and key backup; do not edit encrypted fields or try to bypass the write guards. diff --git a/docs/reference/DATABASE.md b/docs/reference/DATABASE.md index 3110a85..9d38c5c 100644 --- a/docs/reference/DATABASE.md +++ b/docs/reference/DATABASE.md @@ -1,6 +1,6 @@ # Runtime database and migrations -The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `9`. +The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `10`. ## Stored records @@ -16,9 +16,9 @@ The local SQLite database and its sibling artifact root are history and part of - content-addressed blob metadata, logical run/task references, provenance, verification time, and bounded ingestion leases - legacy-run upgrade analysis and the exact task metadata applied by each upgrade -Working memory is stored on the run and in checkpoints. Provider credentials are not stored. Other confidential content may be stored, including prompts, tool output, and remote artifacts. +Working memory is stored on the run and in checkpoints. Provider credentials and state-encryption key values are not stored. Prompts, workflow inputs and outputs, task output and errors, effect input/results, approvals, checkpoints, audit/trace payloads, provider continuations, reconciliation evidence, and long-term-memory values can be protected with application-level authenticated envelopes. -Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. Migration 9 adds retry roots/reason/version and failed-only selection. A repair or retry transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete the derived run. +Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. Migration 9 adds retry roots/reason/version and failed-only selection. Migration 10 adds state-encryption configuration and fail-closed write guards. A repair or retry transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete the derived run. Artifact manifests contain logical path/name, media type, byte size, SHA-256 digest, and CAS-relative path. Blob bytes live under `/artifacts/sha256/`; identical content is stored once. A completed repair/replay receives its own references, so source-row and workspace deletion do not break it. @@ -33,13 +33,16 @@ agentctl runs --db .agentctl/runtime.db analyze RUN_ID --output json agentctl runs --db .agentctl/runtime.db upgrade RUN_ID --dry-run --output json agentctl runs --db .agentctl/runtime.db upgrade RUN_ID --output json agentctl effects --db .agentctl/runtime.db list RUN_ID --output json +agentctl db --db .agentctl/runtime.db encryption inventory --output json +agentctl db --db .agentctl/runtime.db encryption enable --key-id KEY_ID --key-env KEY_ENV --dry-run +agentctl db --db .agentctl/runtime.db encryption rotate --key-id NEW_KEY_ID --key-env NEW_KEY_ENV --dry-run agentctl artifacts --db .agentctl/runtime.db list --run RUN_ID --output json agentctl artifacts --db .agentctl/runtime.db verify --all --output json agentctl artifacts --db .agentctl/runtime.db export SHA256_DIGEST ./report.bin agentctl artifacts --db .agentctl/runtime.db gc --older-than-days 30 --dry-run ``` -`db migrate` may write the database. Back up the database and its WAL state before an upgrade. +`db migrate` and encryption enable/rotation may write the database. Back up the database and its WAL state before an upgrade. An encrypted backup requires the referenced key value; the database stores only the key ID and environment-variable name. Retire pre-encryption backups and snapshots according to their confidentiality requirements. ## Locking and permissions diff --git a/docs/reference/ENVIRONMENT_AND_PATHS.md b/docs/reference/ENVIRONMENT_AND_PATHS.md index 95a4dcc..fddaf1f 100644 --- a/docs/reference/ENVIRONMENT_AND_PATHS.md +++ b/docs/reference/ENVIRONMENT_AND_PATHS.md @@ -12,6 +12,10 @@ These names are defaults used by repository examples. A workflow can name another valid environment reference. Policy must allow the name. Values never belong in YAML, CLI arguments, ordinary inputs, logs, or committed fixtures. +## State-encryption keys + +State encryption accepts an environment-variable reference through `--key-env`. The referenced value must be base64 for exactly 32 bytes. The database stores the key ID and environment-variable name, never the value. Once enabled, every command that opens that database must receive the current reference. Rotation also needs the new reference for that command. + ## Repository and acceptance variables | Variable | Scope | Purpose | diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 4f10c60..07dd989 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2,6 +2,42 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "aes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", +] + +[[package]] +name = "aes-gcm" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", + "zeroize", +] + [[package]] name = "agentctl-core" version = "0.2.0" @@ -87,7 +123,9 @@ dependencies = [ name = "agentctl-store" version = "0.2.0" dependencies = [ + "aes-gcm", "agentctl-core", + "base64", "chrono", "fs2", "hex", @@ -98,6 +136,7 @@ dependencies = [ "sha2", "tempfile", "thiserror", + "zeroize", ] [[package]] @@ -203,6 +242,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -276,12 +324,35 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -310,6 +381,35 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -322,8 +422,8 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", ] [[package]] @@ -560,6 +660,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -631,6 +740,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -827,6 +945,15 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -1127,6 +1254,17 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1911,6 +2049,16 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 4aa5dfd..e725aa8 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] anyhow.workspace = true +base64.workspace = true hex.workspace = true rusqlite.workspace = true serde_json.workspace = true diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 46c9b6a..925b816 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -8,6 +8,8 @@ use std::thread; use std::time::Duration; use anyhow::{Context, Result, bail, ensure}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -15,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 30; +const ACCEPTANCE_SCENARIOS: usize = 31; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1074,6 +1076,156 @@ pub fn run(root: &Path) -> Result<()> { let replay_inspect = inspect(&binary, &workspace, &terminal_retry_db, replay_id)?; ensure!(array_len(&replay_inspect, "/data/effects")? == 0); + scenario( + 31, + "state encryption migrates, rotates, inspects, and replays through the packaged CLI", + ); + let encrypted_db = directory.path().join("encrypted.db"); + let encrypted_source = successful_json( + &binary, + &workspace, + &run_args(&hello, &encrypted_db, &workspace, &[]), + )?; + let encrypted_source_id = string_at(&encrypted_source, "/data/runId")?; + let key_one = STANDARD.encode([31_u8; 32]); + let key_two = STANDARD.encode([47_u8; 32]); + let key_one_env = "AGENTCTL_TEST_STATE_KEY_ONE"; + let key_two_env = "AGENTCTL_TEST_STATE_KEY_TWO"; + let encryption_plan = json_with_env( + &binary, + &workspace, + &strings([ + "db", + "--db", + path(&encrypted_db)?, + "encryption", + "enable", + "--key-id", + "acceptance-key-one", + "--key-env", + key_one_env, + "--dry-run", + "--output", + "json", + "--color", + "never", + ]), + &[(key_one_env, key_one.as_str())], + 0, + )?; + ensure_eq(&encryption_plan, "/data/dryRun", true)?; + let enabled = json_with_env( + &binary, + &workspace, + &strings([ + "db", + "--db", + path(&encrypted_db)?, + "encryption", + "enable", + "--key-id", + "acceptance-key-one", + "--key-env", + key_one_env, + "--output", + "json", + "--color", + "never", + ]), + &[(key_one_env, key_one.as_str())], + 0, + )?; + ensure_eq(&enabled, "/data/operation", "enable")?; + let inventory = json_with_env( + &binary, + &workspace, + &strings([ + "db", + "--db", + path(&encrypted_db)?, + "encryption", + "inventory", + "--output", + "json", + "--color", + "never", + ]), + &[(key_one_env, key_one.as_str())], + 0, + )?; + ensure_eq(&inventory, "/data/enabled", true)?; + ensure_eq(&inventory, "/data/keyId", "acceptance-key-one")?; + ensure_eq(&inventory, "/data/plaintextValues", 0_u64)?; + ensure_eq(&inventory, "/data/invalidEnvelopes", 0_u64)?; + let encrypted_inspect = json_with_env( + &binary, + &workspace, + &strings([ + "inspect", + encrypted_source_id, + "--db", + path(&encrypted_db)?, + "--output", + "json", + "--color", + "never", + ]), + &[(key_one_env, key_one.as_str())], + 0, + )?; + ensure_eq(&encrypted_inspect, "/data/run/runId", encrypted_source_id)?; + let rotated = json_with_env( + &binary, + &workspace, + &strings([ + "db", + "--db", + path(&encrypted_db)?, + "encryption", + "rotate", + "--key-id", + "acceptance-key-two", + "--key-env", + key_two_env, + "--output", + "json", + "--color", + "never", + ]), + &[ + (key_one_env, key_one.as_str()), + (key_two_env, key_two.as_str()), + ], + 0, + )?; + ensure_eq(&rotated, "/data/operation", "rotate")?; + let encrypted_replay = json_with_env( + &binary, + &workspace, + &strings([ + "replay", + encrypted_source_id, + "--db", + path(&encrypted_db)?, + "--output", + "json", + "--color", + "never", + ]), + &[(key_two_env, key_two.as_str())], + 0, + )?; + ensure_eq(&encrypted_replay, "/data/state", "succeeded")?; + let raw_connection = rusqlite::Connection::open(&encrypted_db)?; + let stored_inputs: String = raw_connection.query_row( + "SELECT inputs_json FROM runs WHERE run_id = ?1", + [encrypted_source_id], + |row| row.get(0), + )?; + ensure!(stored_inputs.starts_with("agentctl.encrypted.v1:acceptance-key-two:")); + ensure!(!serde_json::to_string(&inventory)?.contains(&key_one)); + ensure!(!serde_json::to_string(&rotated)?.contains(&key_two)); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } @@ -2238,6 +2390,29 @@ fn json_with_removed_env( parse_output(&output) } +fn json_with_env( + binary: &Path, + cwd: &Path, + args: &[String], + environment: &[(&str, &str)], + code: i32, +) -> Result { + let mut command = command_for(binary, cwd, args); + for name in [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GOOGLE_API_KEY", + "AZURE_OPENAI_API_KEY", + ] { + command.env_remove(name); + } + for (name, value) in environment { + command.env(name, value); + } + let output = output_with_code(command, code, "agentctl with state key reference")?; + parse_output(&output) +} + fn command_for(binary: &Path, cwd: &Path, args: &[String]) -> Command { let mut command = Command::new(binary); command.current_dir(cwd).args(args); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 09450b1..29ad89e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -333,6 +333,10 @@ fn generated_cli_reference(binary: &Path) -> Result { &["artifacts", "export"], &["artifacts", "gc"], &["db"], + &["db", "encryption"], + &["db", "encryption", "inventory"], + &["db", "encryption", "enable"], + &["db", "encryption", "rotate"], &["memory"], &["gc"], &["completion"], @@ -686,6 +690,7 @@ fn verify_public_documentation(root: &Path) -> Result<()> { "docs/guides/FIRST_AGENT_WORKFLOW.md", "docs/guides/WORKFLOW_AUTHORING.md", "docs/guides/LOCAL_OPERATION.md", + "docs/guides/SENSITIVE_STATE_ENCRYPTION.md", "docs/guides/TERMINAL_RETRY.md", "docs/guides/repair-a-failed-workflow.md", "docs/guides/LEGACY_RUN_UPGRADE.md", From 23b0c1b32b5505ee911e134e53a359c203d363cf Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 20:04:45 +0530 Subject: [PATCH 10/44] feat: add stable secret providers --- Cargo.lock | 1 + README.md | 3 +- crates/agentctl-cli/src/main.rs | 485 ++++++++++++++---- crates/agentctl-core/Cargo.toml | 1 + crates/agentctl-core/src/dsl.rs | 265 ++++++++-- crates/agentctl-core/src/lib.rs | 1 + crates/agentctl-core/src/policy.rs | 55 ++ crates/agentctl-core/src/secret.rs | 76 +++ crates/agentctl-protocols/src/lib.rs | 25 +- crates/agentctl-providers/src/lib.rs | 146 ++++-- crates/agentctl-runtime/src/lib.rs | 73 ++- crates/agentctl-runtime/src/process.rs | 9 + crates/agentctl-runtime/src/secret.rs | 391 ++++++++++++++ docs/CONTAINER.md | 39 +- docs/DSL.md | 7 +- docs/OBSERVABILITY.md | 7 +- docs/PROVIDERS.md | 10 +- docs/SECURITY.md | 20 +- docs/THREAT_MODEL.md | 9 +- ...ble-runtime-and-noninteractive-contract.md | 6 +- docs/development/ADD_PROVIDER.md | 5 +- docs/execution/COMPLETENESS_VERIFICATION.md | 2 +- docs/execution/LIMITATION_BURNDOWN.md | 29 +- docs/guides/CI_CD.md | 8 +- docs/guides/SECRET_REFERENCES.md | 140 +++++ docs/guides/TROUBLESHOOTING.md | 14 +- docs/policies.md | 16 +- docs/reference/ENVIRONMENT_AND_PATHS.md | 8 +- docs/reference/YAML.md | 11 +- docs/use-cases/CI_QUALITY_GATE.md | 4 +- fuzz/Cargo.lock | 1 + schemas/workflow.schema.json | 79 ++- xtask/src/acceptance.rs | 88 +++- xtask/src/main.rs | 1 + 34 files changed, 1781 insertions(+), 254 deletions(-) create mode 100644 crates/agentctl-core/src/secret.rs create mode 100644 crates/agentctl-runtime/src/secret.rs create mode 100644 docs/guides/SECRET_REFERENCES.md diff --git a/Cargo.lock b/Cargo.lock index 9436551..33a53cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,7 @@ dependencies = [ "thiserror", "tokio-util", "url", + "zeroize", ] [[package]] diff --git a/README.md b/README.md index 32108dd..dd699b3 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,11 @@ agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md) and [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). For confidential workflow history, use [Sensitive-state encryption](docs/guides/SENSITIVE_STATE_ENCRYPTION.md). +For environment, mounted-file, and policy-gated process credentials, use [Secret references](docs/guides/SECRET_REFERENCES.md). ## Safety boundary -- Secrets are environment references, never inline values or CLI flags. +- Secrets are environment, mounted-file, or policy-gated process references, never inline values or CLI flags. - Files, processes, providers, MCP servers, and A2A peers require explicit policy grants. - Every non-pure operation is recorded before execution. A crash after an at-most-once effect starts is reported as uncertain and is never silently repeated. - Model turns, output tokens, tool calls, retries, and time are bounded. diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index a4f41ef..405f2ba 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::OsString; use std::io::{self, IsTerminal}; use std::path::{Component, Path, PathBuf}; @@ -12,11 +12,13 @@ use agentctl_core::dsl::{ProviderKind, SecretReference, Workflow, parse_workflow use agentctl_core::pack::{PackManifest, verify_pack}; use agentctl_core::policy::PolicyEngine; use agentctl_core::provider::{ContentBlock, Message, ModelProvider, ProviderRequest}; +use agentctl_core::secret::SecretValue; use agentctl_core::{MACHINE_OUTPUT_VERSION, compile}; use agentctl_protocols::{A2aClient, McpClient, ProtocolActionHandler, ProtocolHttpConfig}; use agentctl_providers::{ AnthropicProvider, FakeProvider, GoogleProvider, HttpProviderConfig, OpenAiProvider, }; +use agentctl_runtime::secret::{SecretResolutionError, SecretResolver}; use agentctl_runtime::{ BuiltinToolExecutor, EffectReconciliationInput, RunOptions, Runtime, RuntimeRegistry, }; @@ -700,9 +702,9 @@ async fn execute(cli: Cli) -> Result { .as_deref() .or_else(|| source.base_path.as_deref().map(Path::new)), )?; - let registry = build_registry(&workflow, &base)?; - let runtime = Runtime::new(store, &base).with_registry(registry); let cancellation = cancellation_token(args.timeout_seconds); + let registry = build_registry(&workflow, &base, &cancellation, None).await?; + let runtime = Runtime::new(store, &base).with_registry(registry); let outcome = runtime .fork( &args.run_id, @@ -816,7 +818,7 @@ async fn execute(cli: Cli) -> Result { print_value(output, "RunInspection", &value, Vec::new(), human)?; Ok(EXIT_OK) } - Command::Effects(args) => effect_command(output, args), + Command::Effects(args) => effect_command(output, args).await, Command::Approvals(args) => approval_command(output, args), Command::Providers(args) => provider_command(output, args).await, Command::Auth(args) => auth_command(output, args), @@ -891,10 +893,10 @@ async fn run_workflow(output: OutputFormat, args: RunArgs) -> Result Result>(); + let registry = build_registry(&workflow, &base, &cancellation, Some(&resume_tasks)).await?; + let runtime = Runtime::new(store, &base).with_registry(registry); let outcome = runtime .resume( &args.run_id, @@ -994,9 +1009,10 @@ async fn repair_workflow(output: OutputFormat, args: RepairArgs) -> Result>(); + let registry = build_registry(&workflow, &base, &cancellation, Some(&repair_tasks)).await?; + let runtime = Runtime::new(store, &base).with_registry(registry); let outcome = runtime .repair( &workflow, @@ -1087,9 +1103,10 @@ async fn retry_workflow(output: OutputFormat, args: RetryArgs) -> Result>(); + let registry = build_registry(&workflow, &base, &cancellation, Some(&retry_tasks)).await?; + let runtime = Runtime::new(store, &base).with_registry(registry); let outcome = runtime .retry( &workflow, @@ -1213,7 +1230,7 @@ fn approval_command(output: OutputFormat, args: ApprovalArgs) -> Result Result { +async fn effect_command(output: OutputFormat, args: EffectArgs) -> Result { let store = open_store(&args.db)?; match args.command { EffectCommand::List { run_id, task } => { @@ -1290,7 +1307,10 @@ fn effect_command(output: OutputFormat, args: EffectArgs) -> Result Result { let (workflow, _, diagnostics) = load_and_compile(&args.file)?; + let base = resolve_base_path( + args.file + .parent() + .filter(|path| !path.as_os_str().is_empty()), + )?; + let policy = + PolicyEngine::new(workflow.spec.policy.clone(), &base).map_err(|error| { + CliError { + code: EXIT_POLICY, + message: error.to_string(), + diagnostics: Vec::new(), + run_id: None, + trace_id: None, + } + })?; let data = workflow .spec .providers .iter() .map(|(name, definition)| { + let credential = definition.credential.clone().unwrap_or_else(|| { + SecretReference::environment(default_credential_env( + definition.kind.clone(), + )) + }); serde_json::json!({ "name": name, "kind": definition.kind, @@ -1353,7 +1393,7 @@ async fn provider_command(output: OutputFormat, args: ProviderArgs) -> Result>(), - "credentialConfigured": definition.credential.as_ref().is_some_and(|secret| std::env::var_os(&secret.env).is_some()), + "credential": secret_reference_status(&credential, &policy), }) }) .collect::>(); @@ -1450,17 +1490,31 @@ async fn provider_command(output: OutputFormat, args: ProviderArgs) -> Result Result { let AuthCommand::Check(args) = args.command; let (workflow, _, diagnostics) = load_and_compile(&args.file)?; + let base = resolve_base_path( + args.file + .parent() + .filter(|path| !path.as_os_str().is_empty()), + )?; + let policy = + PolicyEngine::new(workflow.spec.policy.clone(), &base).map_err(|error| CliError { + code: EXIT_POLICY, + message: error.to_string(), + diagnostics: Vec::new(), + run_id: None, + trace_id: None, + })?; let status = workflow .spec .providers .iter() .map(|(name, definition)| { - let env = definition - .credential - .as_ref() - .map(|secret| secret.env.clone()) - .unwrap_or_else(|| default_credential_env(definition.kind.clone()).to_owned()); - serde_json::json!({"provider": name, "environment": env, "present": std::env::var_os(&env).is_some()}) + let reference = definition.credential.clone().unwrap_or_else(|| { + SecretReference::environment(default_credential_env(definition.kind.clone())) + }); + serde_json::json!({ + "provider": name, + "credential": secret_reference_status(&reference, &policy), + }) }) .collect::>(); print_value( @@ -1469,13 +1523,33 @@ fn auth_command(output: OutputFormat, args: AuthArgs) -> Result { &status, diagnostics, format!( - "checked {} credential reference(s); values were not read", + "checked {} credential reference(s); values and secret processes were not read", status.len() ), )?; Ok(EXIT_OK) } +fn secret_reference_status(reference: &SecretReference, policy: &PolicyEngine) -> Value { + match reference { + SecretReference::Environment { env } => serde_json::json!({ + "kind": "environment", + "reference": env, + "availability": if std::env::var_os(env).is_some() { "present" } else { "missing" }, + }), + SecretReference::File { file } => serde_json::json!({ + "kind": "file", + "reference": file, + "availability": if policy.resolve_secret_file(file).is_ok() { "present" } else { "missing_or_denied" }, + }), + SecretReference::Process { process } => serde_json::json!({ + "kind": "process", + "reference": process.command, + "availability": "unchecked", + }), + } +} + fn schema_command(output: OutputFormat, args: SchemaArgs) -> Result { let schema = schema_json(); if let Some(path) = args.write { @@ -2054,34 +2128,70 @@ fn insert_pack_item( } } -fn build_registry(workflow: &Workflow, base: &Path) -> Result { +async fn build_registry( + workflow: &Workflow, + base: &Path, + cancellation: &CancellationToken, + execution_tasks: Option<&BTreeSet>, +) -> Result { let mut registry = RuntimeRegistry::default(); + let tool_policy = + PolicyEngine::new(workflow.spec.policy.clone(), base).map_err(|error| CliError { + code: EXIT_POLICY, + message: error.to_string(), + diagnostics: Vec::new(), + run_id: None, + trace_id: None, + })?; + let provider_secrets = SecretResolver::provider_credentials(tool_policy.clone()); + let restricted_secrets = SecretResolver::restricted(tool_policy.clone()); + let required_providers = workflow + .spec + .tasks + .iter() + .filter(|task| execution_tasks.is_none_or(|selected| selected.contains(&task.id))) + .filter_map(|task| task.uses.strip_prefix("agent:")) + .filter_map(|agent| workflow.spec.agents.get(agent)) + .map(|agent| agent.provider.as_str()) + .collect::>(); + let selected_action_kinds = workflow + .spec + .tasks + .iter() + .filter(|task| execution_tasks.is_none_or(|selected| selected.contains(&task.id))) + .filter_map(|task| task.uses.strip_prefix("action:")) + .filter_map(|action| workflow.spec.actions.get(action)) + .map(|action| action.kind) + .collect::>(); for (name, definition) in &workflow.spec.providers { - let credential = definition - .credential - .clone() - .unwrap_or_else(|| SecretReference { - env: default_credential_env(definition.kind.clone()).to_owned(), - }); - if definition.kind != ProviderKind::Fake && std::env::var_os(&credential.env).is_none() { - return Err(CliError { - code: EXIT_REMOTE, - message: format!( - "provider `{name}` requires environment variable `{}`; configure it or run `agentctl auth check`", - credential.env - ), - diagnostics: Vec::new(), - run_id: None, - trace_id: None, - }); - } + let credential = definition.credential.clone().unwrap_or_else(|| { + SecretReference::environment(default_credential_env(definition.kind.clone())) + }); match definition.kind { ProviderKind::Fake => { registry = registry.with_provider(name, Arc::new(FakeProvider::default())); } ProviderKind::Openai => { - let mut config = HttpProviderConfig::openai(credential.env); - config.headers = resolve_protocol_headers(&definition.headers, workflow)?; + let mut config = + HttpProviderConfig::openai(default_credential_env(ProviderKind::Openai)); + config.credential = credential.clone(); + config.resolved_credential = preflight_provider_credential( + name, + &credential, + &required_providers, + &provider_secrets, + cancellation, + ) + .await?; + config.credential_resolver = Some(Arc::new(provider_secrets.clone())); + if required_providers.contains(name.as_str()) { + config.headers = resolve_protocol_headers( + &definition.headers, + &restricted_secrets, + cancellation, + ) + .await?; + } if let Some(endpoint) = &definition.endpoint { config.endpoint = endpoint.clone(); } @@ -2091,8 +2201,26 @@ fn build_registry(workflow: &Workflow, base: &Path) -> Result { - let mut config = HttpProviderConfig::anthropic(credential.env); - config.headers = resolve_protocol_headers(&definition.headers, workflow)?; + let mut config = + HttpProviderConfig::anthropic(default_credential_env(ProviderKind::Anthropic)); + config.credential = credential.clone(); + config.resolved_credential = preflight_provider_credential( + name, + &credential, + &required_providers, + &provider_secrets, + cancellation, + ) + .await?; + config.credential_resolver = Some(Arc::new(provider_secrets.clone())); + if required_providers.contains(name.as_str()) { + config.headers = resolve_protocol_headers( + &definition.headers, + &restricted_secrets, + cancellation, + ) + .await?; + } if let Some(endpoint) = &definition.endpoint { config.endpoint = endpoint.clone(); } @@ -2102,8 +2230,26 @@ fn build_registry(workflow: &Workflow, base: &Path) -> Result { - let mut config = HttpProviderConfig::google(credential.env); - config.headers = resolve_protocol_headers(&definition.headers, workflow)?; + let mut config = + HttpProviderConfig::google(default_credential_env(ProviderKind::Google)); + config.credential = credential.clone(); + config.resolved_credential = preflight_provider_credential( + name, + &credential, + &required_providers, + &provider_secrets, + cancellation, + ) + .await?; + config.credential_resolver = Some(Arc::new(provider_secrets.clone())); + if required_providers.contains(name.as_str()) { + config.headers = resolve_protocol_headers( + &definition.headers, + &restricted_secrets, + cancellation, + ) + .await?; + } if let Some(endpoint) = &definition.endpoint { config.endpoint = endpoint.clone(); } @@ -2118,16 +2264,35 @@ fn build_registry(workflow: &Workflow, base: &Path) -> Result Result Result Result, + resolver: &SecretResolver, + cancellation: &CancellationToken, +) -> Result, CliError> { + if !required_providers.contains(name) { + return Ok(None); + } + resolver + .resolve(reference, cancellation) + .await + .map(Some) + .map_err(|error| secret_cli_error(name, error, EXIT_REMOTE)) +} + +async fn resolve_protocol_headers( headers: &BTreeMap, - workflow: &Workflow, -) -> Result, CliError> { - headers - .iter() - .map(|(name, reference)| { - if !workflow.spec.policy.environment_allowlist.contains(&reference.env) { - return Err(CliError { - code: EXIT_POLICY, - message: format!( - "header `{name}` secret environment `{}` is not in policy.environmentAllowlist", - reference.env - ), - diagnostics: Vec::new(), - run_id: None, - trace_id: None, - }); - } - let value = std::env::var(&reference.env).map_err(|_| CliError { - code: EXIT_POLICY, - message: format!("required environment variable `{}` is unavailable", reference.env), - diagnostics: Vec::new(), - run_id: None, - trace_id: None, - })?; - Ok((name.clone(), value)) - }) - .collect() + resolver: &SecretResolver, + cancellation: &CancellationToken, +) -> Result, CliError> { + let mut resolved = BTreeMap::new(); + for (name, reference) in headers { + let value = resolver + .resolve(reference, cancellation) + .await + .map_err(|error| secret_cli_error(name, error, EXIT_POLICY))?; + resolved.insert(name.clone(), value); + } + Ok(resolved) +} + +fn secret_cli_error(owner: &str, error: SecretResolutionError, fallback_code: u8) -> CliError { + let code = if matches!(error, SecretResolutionError::Policy(_)) { + EXIT_POLICY + } else { + fallback_code + }; + CliError { + code, + message: format!("secret for `{owner}` could not be resolved: {error}"), + diagnostics: Vec::new(), + run_id: None, + trace_id: None, + } } fn default_credential_env(kind: ProviderKind) -> &'static str { @@ -2558,4 +2740,95 @@ mod tests { assert_eq!(error.code, EXIT_VALIDATION); assert!(error.message.contains("exceeds 1048576 bytes")); } + + #[tokio::test] + async fn registry_preflights_only_reachable_provider_file_credentials() { + let directory = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir(directory.path().join("secrets")).expect("secret directory"); + std::fs::write(directory.path().join("secrets/openai"), "file-secret\n") + .expect("secret file"); + let reachable = parse_workflow( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: reachable-secret } +spec: + policy: + secretFileRoots: [secrets] + networkAllowlist: [api.openai.com] + providers: + openai: + kind: openai + credential: { file: secrets/openai } + agents: + answer: + provider: openai + model: gpt-5.6 + instructions: answer + tasks: + - id: answer + uses: agent:answer + with: { prompt: hello } +"#, + "reachable.yaml", + ) + .expect("reachable workflow") + .workflow; + build_registry( + &reachable, + directory.path(), + &CancellationToken::new(), + None, + ) + .await + .expect("file credential preflight"); + + std::fs::remove_file(directory.path().join("secrets/openai")).expect("remove secret"); + let error = build_registry( + &reachable, + directory.path(), + &CancellationToken::new(), + None, + ) + .await + .err() + .expect("reachable provider requires its credential"); + assert_eq!(error.code, EXIT_POLICY); + assert!(error.message.contains("secret file")); + + let no_tasks = BTreeSet::new(); + build_registry( + &reachable, + directory.path(), + &CancellationToken::new(), + Some(&no_tasks), + ) + .await + .expect("reused provider task does not require its credential"); + + let unused = parse_workflow( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: unused-secret } +spec: + providers: + unused: + kind: openai + credential: { env: AGENTCTL_INTENTIONALLY_MISSING_UNUSED_KEY } + actions: + assign: { kind: builtin.assign } + tasks: + - id: local + uses: action:assign + with: { value: local } +"#, + "unused.yaml", + ) + .expect("unused workflow") + .workflow; + build_registry(&unused, directory.path(), &CancellationToken::new(), None) + .await + .expect("unused provider does not require a credential"); + } } diff --git a/crates/agentctl-core/Cargo.toml b/crates/agentctl-core/Cargo.toml index 80be56d..a3456ee 100644 --- a/crates/agentctl-core/Cargo.toml +++ b/crates/agentctl-core/Cargo.toml @@ -24,6 +24,7 @@ sha2.workspace = true thiserror.workspace = true tokio-util.workspace = true url.workspace = true +zeroize.workspace = true [dev-dependencies] proptest.workspace = true diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index fd8e7a0..dd5da32 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::path::Path; use schemars::{JsonSchema, schema_for}; use serde::{Deserialize, Serialize}; @@ -90,10 +91,55 @@ pub struct ProviderDefinition { pub headers: BTreeMap, } +pub const DEFAULT_SECRET_PROCESS_TIMEOUT_SECONDS: u64 = 5; +pub const DEFAULT_SECRET_OUTPUT_LIMIT_BYTES: u64 = 16 * 1024; +pub const MAX_SECRET_OUTPUT_LIMIT_BYTES: u64 = 64 * 1024; +pub const MAX_SECRET_PROCESS_TIMEOUT_SECONDS: u64 = 60; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(untagged, deny_unknown_fields)] +pub enum SecretReference { + Environment { env: String }, + File { file: String }, + Process { process: SecretProcessReference }, +} + +impl SecretReference { + #[must_use] + pub fn environment(name: impl Into) -> Self { + Self::Environment { env: name.into() } + } + + #[must_use] + pub fn source_description(&self) -> String { + match self { + Self::Environment { env } => format!("environment variable `{env}`"), + Self::File { file } => format!("secret file `{file}`"), + Self::Process { process } => { + format!("secret process `{}`", process.command) + } + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SecretReference { - pub env: String, +pub struct SecretProcessReference { + pub command: String, + #[serde(default)] + pub args: Vec, + #[serde(default = "default_secret_process_timeout_seconds")] + pub timeout_seconds: u64, + #[serde(default = "default_secret_output_limit_bytes")] + pub output_limit_bytes: u64, +} + +const fn default_secret_process_timeout_seconds() -> u64 { + DEFAULT_SECRET_PROCESS_TIMEOUT_SECONDS +} + +const fn default_secret_output_limit_bytes() -> u64 { + DEFAULT_SECRET_OUTPUT_LIMIT_BYTES } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -433,6 +479,10 @@ pub struct PolicyDefinition { #[serde(default)] pub process_allowlist: Vec, #[serde(default)] + pub secret_file_roots: Vec, + #[serde(default)] + pub secret_process_allowlist: Vec, + #[serde(default)] pub providers: Vec, #[serde(default)] pub tools_allow: Vec, @@ -452,6 +502,8 @@ impl Default for PolicyDefinition { environment_allowlist: Vec::new(), network_allowlist: Vec::new(), process_allowlist: Vec::new(), + secret_file_roots: Vec::new(), + secret_process_allowlist: Vec::new(), providers: Vec::new(), tools_allow: Vec::new(), tools_deny: Vec::new(), @@ -871,31 +923,49 @@ fn validate_document(workflow: &Workflow, file: &str) -> Vec { } } for (name, provider) in &workflow.spec.providers { - if let Some(secret) = &provider.credential - && !valid_env_name(&secret.env) - { - diagnostics.push( - Diagnostic::error( - DiagnosticCode::InvalidSecretReference, - file, - format!("provider `{name}` uses invalid environment variable name"), - ) - .with_path(format!("spec.providers.{name}.credential.env")), + if let Some(secret) = &provider.credential { + validate_secret_reference( + secret, + &format!("provider `{name}` credential"), + &format!("spec.providers.{name}.credential"), + &workflow.spec.policy, + file, + &mut diagnostics, ); } for (header, secret) in &provider.headers { - if !valid_env_name(&secret.env) { - diagnostics.push( - Diagnostic::error( - DiagnosticCode::InvalidSecretReference, - file, - format!( - "provider `{name}` header `{header}` uses an invalid secret reference" - ), - ) - .with_path(format!("spec.providers.{name}.headers.{header}.env")), - ); - } + validate_secret_reference( + secret, + &format!("provider `{name}` header `{header}`"), + &format!("spec.providers.{name}.headers.{header}"), + &workflow.spec.policy, + file, + &mut diagnostics, + ); + } + } + for (name, action) in &workflow.spec.actions { + for (environment, secret) in &action.env { + validate_secret_reference( + secret, + &format!("action `{name}` environment `{environment}`"), + &format!("spec.actions.{name}.env.{environment}"), + &workflow.spec.policy, + file, + &mut diagnostics, + ); + } + } + for (name, tool) in &workflow.spec.tools { + for (position, secret) in tool.secrets.iter().enumerate() { + validate_secret_reference( + secret, + &format!("tool `{name}` secret {position}"), + &format!("spec.tools.{name}.secrets[{position}]"), + &workflow.spec.policy, + file, + &mut diagnostics, + ); } } for (name, agent) in &workflow.spec.agents { @@ -970,39 +1040,98 @@ fn validate_document(workflow: &Workflow, file: &str) -> Vec { } for (name, server) in &workflow.spec.mcp_servers { for (header, secret) in &server.headers { - if !valid_env_name(&secret.env) { - diagnostics.push( - Diagnostic::error( - DiagnosticCode::InvalidSecretReference, - file, - format!( - "MCP server `{name}` header `{header}` has an invalid secret reference" - ), - ) - .with_path(format!("spec.mcpServers.{name}.headers.{header}.env")), - ); - } + validate_secret_reference( + secret, + &format!("MCP server `{name}` header `{header}`"), + &format!("spec.mcpServers.{name}.headers.{header}"), + &workflow.spec.policy, + file, + &mut diagnostics, + ); } } for (name, peer) in &workflow.spec.a2a_peers { for (header, secret) in &peer.headers { - if !valid_env_name(&secret.env) { - diagnostics.push( - Diagnostic::error( - DiagnosticCode::InvalidSecretReference, - file, - format!( - "A2A peer `{name}` header `{header}` has an invalid secret reference" - ), - ) - .with_path(format!("spec.a2aPeers.{name}.headers.{header}.env")), - ); - } + validate_secret_reference( + secret, + &format!("A2A peer `{name}` header `{header}`"), + &format!("spec.a2aPeers.{name}.headers.{header}"), + &workflow.spec.policy, + file, + &mut diagnostics, + ); } } diagnostics } +fn validate_secret_reference( + reference: &SecretReference, + label: &str, + path: &str, + policy: &PolicyDefinition, + file: &str, + diagnostics: &mut Vec, +) { + let error = match reference { + SecretReference::Environment { env } if !valid_env_name(env) => { + Some(format!("{label} uses an invalid environment variable name")) + } + SecretReference::File { file } if file.trim().is_empty() || file.contains('\0') => { + Some(format!("{label} uses an invalid secret file path")) + } + SecretReference::File { .. } if policy.secret_file_roots.is_empty() => Some(format!( + "{label} requires at least one policy.secretFileRoots entry" + )), + SecretReference::Process { process } if process.command.trim().is_empty() => { + Some(format!("{label} uses an empty secret process command")) + } + SecretReference::Process { process } + if process.args.len() > 64 + || process.args.iter().any(|argument| argument.len() > 4096) => + { + Some(format!( + "{label} secret process accepts at most 64 arguments of 4096 bytes each" + )) + } + SecretReference::Process { process } + if process.timeout_seconds == 0 + || process.timeout_seconds > MAX_SECRET_PROCESS_TIMEOUT_SECONDS => + { + Some(format!( + "{label} secret process timeoutSeconds must be between 1 and {MAX_SECRET_PROCESS_TIMEOUT_SECONDS}" + )) + } + SecretReference::Process { process } + if process.output_limit_bytes == 0 + || process.output_limit_bytes > MAX_SECRET_OUTPUT_LIMIT_BYTES => + { + Some(format!( + "{label} secret process outputLimitBytes must be between 1 and {MAX_SECRET_OUTPUT_LIMIT_BYTES}" + )) + } + SecretReference::Process { process } + if !policy.secret_process_allowlist.iter().any(|allowed| { + Path::new(&process.command) + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|basename| basename == allowed) + }) => + { + Some(format!( + "{label} process is not in policy.secretProcessAllowlist" + )) + } + _ => None, + }; + if let Some(error) = error { + diagnostics.push( + Diagnostic::error(DiagnosticCode::InvalidSecretReference, file, error) + .with_path(path.to_owned()), + ); + } +} + fn valid_env_name(name: &str) -> bool { let mut chars = name.chars(); let Some(first) = chars.next() else { @@ -1068,6 +1197,46 @@ spec: assert_eq!(diagnostics[0].code, DiagnosticCode::InvalidSecretReference); } + #[test] + fn parses_bounded_file_and_process_secret_references() { + let source = MINIMAL.replace( + " actions:", + " policy:\n secretFileRoots: [secrets]\n secretProcessAllowlist: [secret-helper]\n providers:\n file:\n kind: openai\n credential: { file: secrets/openai }\n process:\n kind: anthropic\n credential:\n process:\n command: /usr/local/bin/secret-helper\n args: [read, anthropic]\n timeoutSeconds: 3\n outputLimitBytes: 128\n actions:", + ); + let workflow = parse_workflow(&source, "secrets.yaml") + .expect("valid secret references") + .workflow; + assert!(matches!( + workflow.spec.providers["file"].credential.as_ref(), + Some(SecretReference::File { .. }) + )); + assert!(matches!( + workflow.spec.providers["process"].credential.as_ref(), + Some(SecretReference::Process { .. }) + )); + } + + #[test] + fn secret_file_and_process_references_require_explicit_policy() { + let file_source = MINIMAL.replace( + " actions:", + " providers:\n openai:\n kind: openai\n credential: { file: /run/secrets/openai }\n actions:", + ); + let diagnostics = + parse_workflow(&file_source, "bad.yaml").expect_err("missing secret root"); + assert_eq!(diagnostics[0].code, DiagnosticCode::InvalidSecretReference); + assert!(diagnostics[0].message.contains("secretFileRoots")); + + let process_source = MINIMAL.replace( + " actions:", + " providers:\n openai:\n kind: openai\n credential:\n process:\n command: secret-helper\n timeoutSeconds: 0\n actions:", + ); + let diagnostics = + parse_workflow(&process_source, "bad.yaml").expect_err("invalid secret process"); + assert_eq!(diagnostics[0].code, DiagnosticCode::InvalidSecretReference); + assert!(diagnostics[0].message.contains("timeoutSeconds")); + } + #[test] fn rejects_process_limits_on_non_process_actions() { let source = MINIMAL.replace( diff --git a/crates/agentctl-core/src/lib.rs b/crates/agentctl-core/src/lib.rs index 1f2f798..5a4bcd4 100644 --- a/crates/agentctl-core/src/lib.rs +++ b/crates/agentctl-core/src/lib.rs @@ -7,6 +7,7 @@ pub mod effect; pub mod pack; pub mod policy; pub mod provider; +pub mod secret; pub mod state; pub mod template; pub mod tool; diff --git a/crates/agentctl-core/src/policy.rs b/crates/agentctl-core/src/policy.rs index b7a3ef9..528280a 100644 --- a/crates/agentctl-core/src/policy.rs +++ b/crates/agentctl-core/src/policy.rs @@ -16,6 +16,7 @@ pub struct PolicyEngine { policy: PolicyDefinition, workspace_root: PathBuf, writable_roots: Vec, + secret_file_roots: Vec, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -54,6 +55,10 @@ pub enum PolicyError { EnvironmentDenied(String), #[error("process is not authorized: {0}")] ProcessDenied(String), + #[error("secret file is not authorized: {0}")] + SecretFileDenied(String), + #[error("secret process is not authorized: {0}")] + SecretProcessDenied(String), } impl PolicyEngine { @@ -76,10 +81,23 @@ impl PolicyEngine { canonicalize_existing_or_parent(&candidate) }) .collect::, _>>()?; + let secret_file_roots = policy + .secret_file_roots + .iter() + .map(|root| { + let candidate = if Path::new(root).is_absolute() { + PathBuf::from(root) + } else { + workspace_root.join(root) + }; + canonicalize_existing_or_parent(&candidate) + }) + .collect::, _>>()?; Ok(Self { policy, workspace_root, writable_roots, + secret_file_roots, }) } @@ -156,6 +174,11 @@ impl PolicyEngine { } } + #[must_use] + pub fn workspace_root(&self) -> &Path { + &self.workspace_root + } + #[must_use] pub fn decide_with_approval( &self, @@ -287,6 +310,38 @@ impl PolicyEngine { } } + pub fn authorize_secret_process(&self, command: &str) -> Result<(), PolicyError> { + let basename = Path::new(command) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(command); + if self + .policy + .secret_process_allowlist + .iter() + .any(|allowed| allowed == basename) + { + Ok(()) + } else { + Err(PolicyError::SecretProcessDenied(command.to_owned())) + } + } + + pub fn resolve_secret_file(&self, requested: &str) -> Result { + let candidate = self.join_workspace(requested)?; + let canonical = fs::canonicalize(&candidate) + .map_err(|_| PolicyError::SecretFileDenied(requested.to_owned()))?; + if !canonical.is_file() + || !self + .secret_file_roots + .iter() + .any(|root| canonical.starts_with(root)) + { + return Err(PolicyError::SecretFileDenied(requested.to_owned())); + } + Ok(canonical) + } + #[must_use] pub fn filtered_environment( &self, diff --git a/crates/agentctl-core/src/secret.rs b/crates/agentctl-core/src/secret.rs new file mode 100644 index 0000000..7940e03 --- /dev/null +++ b/crates/agentctl-core/src/secret.rs @@ -0,0 +1,76 @@ +use std::fmt; + +use async_trait::async_trait; +use tokio_util::sync::CancellationToken; +use zeroize::Zeroizing; + +use crate::dsl::SecretReference; + +/// A resolved secret held only in memory and zeroized when dropped. +pub struct SecretValue(Zeroizing); + +impl SecretValue { + #[must_use] + pub fn new(value: String) -> Self { + Self(Zeroizing::new(value)) + } + + #[must_use] + pub fn expose(&self) -> &str { + self.0.as_str() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl Clone for SecretValue { + fn clone(&self) -> Self { + Self::new(self.expose().to_owned()) + } +} + +impl fmt::Debug for SecretValue { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretValue([REDACTED])") + } +} + +impl From for SecretValue { + fn from(value: String) -> Self { + Self::new(value) + } +} + +impl From<&str> for SecretValue { + fn from(value: &str) -> Self { + Self::new(value.to_owned()) + } +} + +/// Runtime-supplied secret resolution used by provider adapters at dispatch time. +#[async_trait] +pub trait SecretSourceResolver: fmt::Debug + Send + Sync { + /// Resolve a reference without persisting or logging the returned value. + /// + /// Error messages must describe only the source and failure, never the value. + async fn resolve_secret( + &self, + reference: &SecretReference, + cancellation: &CancellationToken, + ) -> Result; +} + +#[cfg(test)] +mod tests { + use super::SecretValue; + + #[test] + fn debug_never_exposes_the_value() { + let value = SecretValue::from("fixture-secret"); + assert_eq!(format!("{value:?}"), "SecretValue([REDACTED])"); + assert_eq!(value.expose(), "fixture-secret"); + } +} diff --git a/crates/agentctl-protocols/src/lib.rs b/crates/agentctl-protocols/src/lib.rs index 37aa0e8..5da5521 100644 --- a/crates/agentctl-protocols/src/lib.rs +++ b/crates/agentctl-protocols/src/lib.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use agentctl_core::dsl::ActionKind; +use agentctl_core::secret::SecretValue; use agentctl_runtime::{ExternalActionHandler, RuntimeError}; use async_trait::async_trait; use futures_util::StreamExt; @@ -50,7 +51,7 @@ pub enum ProtocolError { #[derive(Debug, Clone)] pub struct ProtocolHttpConfig { pub url: Url, - pub headers: BTreeMap, + pub headers: BTreeMap, pub timeout: Duration, } @@ -262,7 +263,7 @@ impl McpClient { .post(self.config.url.clone()) .header("Accept", "application/json, text/event-stream") .header("Origin", "agentctl://local"), - |request, (name, value)| request.header(name, value), + |request, (name, value)| request.header(name, value.expose()), ) } @@ -351,7 +352,7 @@ impl A2aClient { ) -> Result { let request = self.card_config.headers.iter().fold( self.client.get(self.card_config.url.clone()), - |request, (name, value)| request.header(name, value), + |request, (name, value)| request.header(name, value.expose()), ); let response = execute_request(request, self.card_config.timeout, cancellation, None).await?; @@ -557,7 +558,7 @@ impl A2aClient { .post(url) .header("A2A-Version", A2A_PROTOCOL_VERSION) .header("Content-Type", "application/json"), - |request, (name, value)| request.header(name, value), + |request, (name, value)| request.header(name, value.expose()), )) } } @@ -683,7 +684,7 @@ async fn response_json Deserialize<'de>>( response: Response, timeout: Duration, cancellation: &CancellationToken, - headers: &BTreeMap, + headers: &BTreeMap, ) -> Result { if !response.status().is_success() { return Err(http_error(response).await); @@ -699,7 +700,7 @@ async fn response_value( response: Response, timeout: Duration, cancellation: &CancellationToken, - headers: &BTreeMap, + headers: &BTreeMap, ) -> Result { response_values(response, timeout, cancellation, headers) .await? @@ -712,7 +713,7 @@ async fn response_values( response: Response, timeout: Duration, cancellation: &CancellationToken, - headers: &BTreeMap, + headers: &BTreeMap, ) -> Result, ProtocolError> { if !response.status().is_success() { return Err(http_error(response).await); @@ -747,11 +748,11 @@ async fn response_values( } } -fn redact_header_secrets(value: &mut Value, headers: &BTreeMap) { +fn redact_header_secrets(value: &mut Value, headers: &BTreeMap) { match value { Value::String(text) => { for secret in headers.values().filter(|secret| !secret.is_empty()) { - *text = text.replace(secret, "[REDACTED]"); + *text = text.replace(secret.expose(), "[REDACTED]"); } } Value::Array(values) => { @@ -766,7 +767,9 @@ fn redact_header_secrets(value: &mut Value, headers: &BTreeMap) let name = headers .values() .filter(|secret| !secret.is_empty()) - .fold(name, |name, secret| name.replace(secret, "[REDACTED]")); + .fold(name, |name, secret| { + name.replace(secret.expose(), "[REDACTED]") + }); values.insert(name, value); } } @@ -914,7 +917,7 @@ mod tests { .await; let client = McpClient::new(ProtocolHttpConfig { url: Url::parse(&format!("{}/mcp", server.uri())).expect("url"), - headers: BTreeMap::from([("authorization".to_owned(), "Bearer fixture".to_owned())]), + headers: BTreeMap::from([("authorization".to_owned(), "Bearer fixture".into())]), timeout: Duration::from_secs(2), }) .expect("client"); diff --git a/crates/agentctl-providers/src/lib.rs b/crates/agentctl-providers/src/lib.rs index a2a6d6c..ce79ba5 100644 --- a/crates/agentctl-providers/src/lib.rs +++ b/crates/agentctl-providers/src/lib.rs @@ -1,13 +1,14 @@ //! Native provider adapters for agentctl. use std::collections::{BTreeMap, VecDeque}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use agentctl_core::dsl::{ReasoningEffort, SecretReference}; use agentctl_core::provider::{ ContentBlock, ContinuationState, FinishReason, Message, ModelProvider, ProviderError, ProviderRequest, ProviderResponse, ToolCall, Usage, }; +use agentctl_core::secret::{SecretSourceResolver, SecretValue}; use async_trait::async_trait; use futures_util::StreamExt; use reqwest::{Client, StatusCode}; @@ -21,10 +22,12 @@ const MAX_PROVIDER_RESPONSE_BYTES: usize = 4 * 1024 * 1024; pub struct HttpProviderConfig { pub endpoint: String, pub credential: SecretReference, + pub resolved_credential: Option, + pub credential_resolver: Option>, pub organization: Option, pub project: Option, pub api_version: Option, - pub headers: BTreeMap, + pub headers: BTreeMap, } impl HttpProviderConfig { @@ -32,9 +35,9 @@ impl HttpProviderConfig { pub fn openai(credential_env: impl Into) -> Self { Self { endpoint: "https://api.openai.com/v1/responses".to_owned(), - credential: SecretReference { - env: credential_env.into(), - }, + credential: SecretReference::environment(credential_env), + resolved_credential: None, + credential_resolver: None, organization: None, project: None, api_version: None, @@ -46,9 +49,9 @@ impl HttpProviderConfig { pub fn anthropic(credential_env: impl Into) -> Self { Self { endpoint: "https://api.anthropic.com/v1/messages".to_owned(), - credential: SecretReference { - env: credential_env.into(), - }, + credential: SecretReference::environment(credential_env), + resolved_credential: None, + credential_resolver: None, organization: None, project: None, api_version: None, @@ -60,9 +63,9 @@ impl HttpProviderConfig { pub fn google(credential_env: impl Into) -> Self { Self { endpoint: "https://generativelanguage.googleapis.com/v1beta/models".to_owned(), - credential: SecretReference { - env: credential_env.into(), - }, + credential: SecretReference::environment(credential_env), + resolved_credential: None, + credential_resolver: None, organization: None, project: None, api_version: None, @@ -115,7 +118,7 @@ impl ModelProvider for OpenAiProvider { request: &ProviderRequest, cancellation: &CancellationToken, ) -> Result { - let credential = load_credential(&self.config.credential)?; + let credential = load_credential(&self.config, cancellation).await?; let endpoint = if self.azure { let separator = if self.config.endpoint.contains('?') { '&' @@ -132,12 +135,12 @@ impl ModelProvider for OpenAiProvider { }; let mut http = self.client.post(endpoint).json(&openai_request(request)?); for (name, value) in &self.config.headers { - http = http.header(name, value); + http = http.header(name, value.expose()); } http = if self.azure { - http.header("api-key", &credential) + http.header("api-key", credential.expose()) } else { - http.bearer_auth(&credential) + http.bearer_auth(credential.expose()) }; if let Some(organization) = &self.config.organization { http = http.header("OpenAI-Organization", organization); @@ -458,7 +461,7 @@ impl ModelProvider for AnthropicProvider { request: &ProviderRequest, cancellation: &CancellationToken, ) -> Result { - let credential = load_credential(&self.config.credential)?; + let credential = load_credential(&self.config, cancellation).await?; let http = self .client .post(&self.config.endpoint) @@ -467,9 +470,11 @@ impl ModelProvider for AnthropicProvider { .config .headers .iter() - .fold(http, |request, (name, value)| request.header(name, value)); + .fold(http, |request, (name, value)| { + request.header(name, value.expose()) + }); let http = http - .header("x-api-key", &credential) + .header("x-api-key", credential.expose()) .header("anthropic-version", ANTHROPIC_VERSION); let secrets = configured_secrets(&credential, &self.config.headers); let response = send(http, cancellation, &secrets).await?; @@ -664,7 +669,7 @@ impl ModelProvider for GoogleProvider { request: &ProviderRequest, cancellation: &CancellationToken, ) -> Result { - let credential = load_credential(&self.config.credential)?; + let credential = load_credential(&self.config, cancellation).await?; let endpoint = format!( "{}/{}:generateContent", self.config.endpoint.trim_end_matches('/'), @@ -675,8 +680,10 @@ impl ModelProvider for GoogleProvider { .config .headers .iter() - .fold(http, |request, (name, value)| request.header(name, value)); - let http = http.header("x-goog-api-key", &credential); + .fold(http, |request, (name, value)| { + request.header(name, value.expose()) + }); + let http = http.header("x-goog-api-key", credential.expose()); let secrets = configured_secrets(&credential, &self.config.headers); let response = send(http, cancellation, &secrets).await?; parse_google(response, request) @@ -1064,11 +1071,11 @@ async fn send( } fn configured_secrets<'a>( - credential: &'a str, - headers: &'a BTreeMap, + credential: &'a SecretValue, + headers: &'a BTreeMap, ) -> Vec<&'a str> { - std::iter::once(credential) - .chain(headers.values().map(String::as_str)) + std::iter::once(credential.expose()) + .chain(headers.values().map(SecretValue::expose)) .filter(|secret| !secret.is_empty()) .collect() } @@ -1114,12 +1121,35 @@ fn normalize_transport(error: reqwest::Error) -> ProviderError { } } -fn load_credential(reference: &SecretReference) -> Result { +async fn load_credential( + config: &HttpProviderConfig, + cancellation: &CancellationToken, +) -> Result { + if let Some(credential) = &config.resolved_credential { + return Ok(credential.clone()); + } + if let Some(resolver) = &config.credential_resolver { + return resolver + .resolve_secret(&config.credential, cancellation) + .await + .map_err(ProviderError::Authentication); + } #[cfg(test)] - if reference.env == "AGENTCTL_PROVIDER_TEST_KEY" { - return Ok("test-key".to_owned()); + if matches!( + &config.credential, + SecretReference::Environment { env } if env == "AGENTCTL_PROVIDER_TEST_KEY" + ) { + return Ok(SecretValue::from("test-key")); + } + match &config.credential { + SecretReference::Environment { env } => std::env::var(env) + .map(SecretValue::new) + .map_err(|_| ProviderError::Authentication(env.clone())), + reference => Err(ProviderError::Authentication(format!( + "{} was not resolved by the runtime", + reference.source_description() + ))), } - std::env::var(&reference.env).map_err(|_| ProviderError::Authentication(reference.env.clone())) } fn required_field(value: &Value, field: &str) -> Result { @@ -1440,9 +1470,9 @@ mod tests { .await; let config = HttpProviderConfig { endpoint: format!("{}/openai/v1/responses", server.uri()), - credential: SecretReference { - env: "AGENTCTL_PROVIDER_TEST_KEY".to_owned(), - }, + credential: SecretReference::environment("AGENTCTL_PROVIDER_TEST_KEY"), + resolved_credential: None, + credential_resolver: None, organization: None, project: None, api_version: Some("v1".to_owned()), @@ -1474,7 +1504,7 @@ mod tests { config.endpoint = format!("{}/v1/responses", server.uri()); config .headers - .insert("x-custom-auth".to_owned(), "header-secret".to_owned()); + .insert("x-custom-auth".to_owned(), "header-secret".into()); let error = OpenAiProvider::new(config) .expect("provider") .complete(&request(), &CancellationToken::new()) @@ -1516,7 +1546,7 @@ mod tests { config.endpoint = format!("{}/v1/responses", server.uri()); config .headers - .insert("x-custom-auth".to_owned(), "header-secret".to_owned()); + .insert("x-custom-auth".to_owned(), "header-secret".into()); let response = OpenAiProvider::new(config) .expect("provider") .complete(&request(), &CancellationToken::new()) @@ -1528,6 +1558,54 @@ mod tests { assert_eq!(response.text, "echo [REDACTED] and [REDACTED]"); } + #[tokio::test] + async fn runtime_resolved_non_environment_credential_is_used_and_redacted() { + #[derive(Debug)] + struct FixtureSecretResolver; + + #[async_trait] + impl SecretSourceResolver for FixtureSecretResolver { + async fn resolve_secret( + &self, + _reference: &SecretReference, + _cancellation: &CancellationToken, + ) -> Result { + Ok(SecretValue::from("resolved-file-secret")) + } + } + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(header("authorization", "Bearer resolved-file-secret")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "id": "resp_resolved-file-secret", + "status": "completed", + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "resolved-file-secret"}]} + ] + }))) + .mount(&server) + .await; + let mut config = HttpProviderConfig::openai("unused"); + config.endpoint = format!("{}/v1/responses", server.uri()); + config.credential = SecretReference::File { + file: "/run/secrets/openai".to_owned(), + }; + config.credential_resolver = Some(Arc::new(FixtureSecretResolver)); + let response = OpenAiProvider::new(config) + .expect("provider") + .complete(&request(), &CancellationToken::new()) + .await + .expect("response"); + assert_eq!(response.text, "[REDACTED]"); + assert!( + !serde_json::to_string(&response) + .expect("response json") + .contains("resolved-file-secret") + ); + } + #[tokio::test] async fn rate_limits_are_explicitly_retryable() { let server = MockServer::start().await; diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index 91030ce..87567ac 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -40,6 +40,7 @@ use url::Url; use uuid::Uuid; mod process; +pub mod secret; use process::{ProcessOutputLimits, ProcessRunError, run_bounded_process}; @@ -3049,20 +3050,18 @@ impl Runtime { .unwrap_or_else(|| self.base_path.clone()); let mut resolved_environment = BTreeMap::new(); let mut environment_digests = BTreeMap::new(); + let secret_resolver = secret::SecretResolver::restricted(policy.clone()); for (name, reference) in &action.env { policy.authorize_environment(name)?; - policy.authorize_environment(&reference.env)?; - let value = std::env::var(&reference.env).map_err(|_| { - RuntimeError::InvalidState(format!( - "required environment variable `{}` is unavailable", - reference.env - )) - })?; + let value = secret_resolver + .resolve(reference, cancellation) + .await + .map_err(|error| RuntimeError::InvalidState(error.to_string()))?; environment_digests.insert( name.clone(), serde_json::json!({ - "source": reference.env, - "valueDigest": digest(value.as_bytes()), + "source": reference.source_description(), + "valueDigest": digest(value.expose().as_bytes()), }), ); resolved_environment.insert(name.clone(), value); @@ -3111,7 +3110,7 @@ impl Runtime { .env_clear() .kill_on_drop(true); for (name, value) in &resolved_environment { - process.env(name, value); + process.env(name, value.expose()); } let timeout = Duration::from_secs( action @@ -3129,7 +3128,7 @@ impl Runtime { Ok(result) => { let secrets = resolved_environment .values() - .map(String::as_str) + .map(agentctl_core::secret::SecretValue::expose) .collect::>(); let output = serde_json::json!({ "status": if result.status.success() {"changed"} else {"failed"}, @@ -3176,7 +3175,7 @@ impl Runtime { }) => { let secrets = resolved_environment .values() - .map(String::as_str) + .map(agentctl_core::secret::SecretValue::expose) .collect::>(); let diagnostic = serde_json::json!({ "code": "subprocess_output_limit_exceeded", @@ -8873,8 +8872,15 @@ spec: #[cfg(unix)] #[tokio::test] - async fn normal_subprocess_success_redacts_secret_output() { + async fn file_secret_subprocess_output_is_redacted_and_never_persisted() { let directory = tempdir().expect("tempdir"); + std::fs::create_dir(directory.path().join("secrets")).expect("secret directory"); + let secret = "mounted-file-secret-marker"; + std::fs::write( + directory.path().join("secrets/token"), + format!("{secret}\n"), + ) + .expect("secret file"); let (workflow, plan) = compile_fixture( r#" apiVersion: agentctl.dev/v1alpha1 @@ -8884,7 +8890,8 @@ spec: policy: workspaceRoot: . processAllowlist: [sh] - environmentAllowlist: [SECRET, PATH] + environmentAllowlist: [SECRET] + secretFileRoots: [secrets] approval: never actions: print: @@ -8892,11 +8899,13 @@ spec: command: /bin/sh args: [-c, 'printf "%s" "$SECRET"'] env: - SECRET: { env: PATH } + SECRET: { file: secrets/token } tasks: [{ id: print, uses: "action:print" }] "#, ); - let outcome = runtime(SqliteStore::open_memory().expect("store"), directory.path()) + let database = directory.path().join("runtime.db"); + let store = SqliteStore::open(&database).expect("store"); + let outcome = runtime(store.clone(), directory.path()) .start( &workflow, &plan, @@ -8911,6 +8920,38 @@ spec: outcome.output.as_ref().expect("output")["print"]["stdout"], "[REDACTED]" ); + let effect = &store.list_effects(&outcome.run_id).expect("effects")[0]; + assert_eq!( + effect.request.input["environment"]["SECRET"]["source"], + "secret file `secrets/token`" + ); + assert!( + !serde_json::to_string(effect) + .expect("effect json") + .contains(secret) + ); + let raw = rusqlite::Connection::open(database).expect("raw database"); + let occurrences: i64 = raw + .query_row( + "SELECT COUNT(*) FROM ( + SELECT workflow_json AS value FROM runs + UNION ALL SELECT inputs_json FROM runs + UNION ALL SELECT working_memory_json FROM runs + UNION ALL SELECT output_json FROM runs WHERE output_json IS NOT NULL + UNION ALL SELECT output_json FROM task_states WHERE output_json IS NOT NULL + UNION ALL SELECT error FROM task_states WHERE error IS NOT NULL + UNION ALL SELECT input_json FROM effects + UNION ALL SELECT result_json FROM effects WHERE result_json IS NOT NULL + UNION ALL SELECT error FROM effects WHERE error IS NOT NULL + UNION ALL SELECT state_json FROM checkpoints + UNION ALL SELECT payload_json FROM audit_events + UNION ALL SELECT event_json FROM trace_events + ) WHERE instr(value, ?1) > 0", + [secret], + |row| row.get(0), + ) + .expect("secret scan"); + assert_eq!(occurrences, 0); } #[cfg(unix)] diff --git a/crates/agentctl-runtime/src/process.rs b/crates/agentctl-runtime/src/process.rs index 788f86f..92c5ff0 100644 --- a/crates/agentctl-runtime/src/process.rs +++ b/crates/agentctl-runtime/src/process.rs @@ -51,6 +51,15 @@ pub enum ProcessRunError { }, } +impl ProcessRunError { + pub(crate) fn clear_captured_output(&mut self) { + if let Self::OutputLimitExceeded { stdout, stderr, .. } = self { + stdout.fill(0); + stderr.fill(0); + } + } +} + #[derive(Debug, Clone, Copy)] enum Stream { Stdout, diff --git a/crates/agentctl-runtime/src/secret.rs b/crates/agentctl-runtime/src/secret.rs new file mode 100644 index 0000000..d065442 --- /dev/null +++ b/crates/agentctl-runtime/src/secret.rs @@ -0,0 +1,391 @@ +use std::time::Duration; + +use agentctl_core::dsl::{MAX_SECRET_OUTPUT_LIMIT_BYTES, SecretReference}; +use agentctl_core::policy::{PolicyEngine, PolicyError}; +use agentctl_core::secret::{SecretSourceResolver, SecretValue}; +use async_trait::async_trait; +use thiserror::Error; +use tokio::io::AsyncReadExt; +use tokio::process::Command; +use tokio_util::sync::CancellationToken; + +use crate::process::{ProcessOutputLimits, ProcessRunError, run_bounded_process}; + +const MAX_SECRET_FILE_BYTES: u64 = MAX_SECRET_OUTPUT_LIMIT_BYTES; +const SECRET_PROCESS_STDERR_LIMIT_BYTES: u64 = 16 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnvironmentSecretPolicy { + RequireAllowlist, + ProviderCredentialCompatibility, +} + +#[derive(Debug, Clone)] +pub struct SecretResolver { + policy: PolicyEngine, + environment_policy: EnvironmentSecretPolicy, +} + +impl SecretResolver { + #[must_use] + pub fn restricted(policy: PolicyEngine) -> Self { + Self { + policy, + environment_policy: EnvironmentSecretPolicy::RequireAllowlist, + } + } + + #[must_use] + pub fn provider_credentials(policy: PolicyEngine) -> Self { + Self { + policy, + environment_policy: EnvironmentSecretPolicy::ProviderCredentialCompatibility, + } + } + + pub async fn resolve( + &self, + reference: &SecretReference, + cancellation: &CancellationToken, + ) -> Result { + let value = match reference { + SecretReference::Environment { env } => { + if self.environment_policy == EnvironmentSecretPolicy::RequireAllowlist { + self.policy.authorize_environment(env)?; + } + std::env::var(env).map_err(|_| { + SecretResolutionError::Unavailable(reference.source_description()) + })? + } + SecretReference::File { file } => { + let path = self.policy.resolve_secret_file(file)?; + let input = tokio::fs::File::open(&path).await.map_err(|_| { + SecretResolutionError::Unavailable(reference.source_description()) + })?; + let mut bytes = Vec::new(); + input + .take(MAX_SECRET_FILE_BYTES.saturating_add(1)) + .read_to_end(&mut bytes) + .await + .map_err(|_| { + SecretResolutionError::Unavailable(reference.source_description()) + })?; + if bytes.len() as u64 > MAX_SECRET_FILE_BYTES { + bytes.fill(0); + return Err(SecretResolutionError::OutputLimit { + reference: reference.source_description(), + limit_bytes: MAX_SECRET_FILE_BYTES, + }); + } + secret_utf8(bytes, reference)? + } + SecretReference::Process { process } => { + if cancellation.is_cancelled() { + return Err(SecretResolutionError::Process(ProcessRunError::Cancelled)); + } + self.policy.authorize_secret_process(&process.command)?; + let mut command = Command::new(&process.command); + command + .args(&process.args) + .current_dir(self.policy.workspace_root()) + .env_clear() + .kill_on_drop(true); + let mut output = match run_bounded_process( + command, + ProcessOutputLimits { + stdout_bytes: process.output_limit_bytes, + stderr_bytes: SECRET_PROCESS_STDERR_LIMIT_BYTES, + combined_bytes: process + .output_limit_bytes + .saturating_add(SECRET_PROCESS_STDERR_LIMIT_BYTES), + }, + Duration::from_secs(process.timeout_seconds), + cancellation, + ) + .await + { + Ok(output) => output, + Err(mut error) => { + error.clear_captured_output(); + return Err(error.into()); + } + }; + output.stderr.fill(0); + if !output.status.success() { + output.stdout.fill(0); + return Err(SecretResolutionError::ProcessExit { + reference: reference.source_description(), + code: output.status.code(), + }); + } + secret_utf8(output.stdout, reference)? + } + }; + let value = strip_one_line_ending(value); + if value.is_empty() { + return Err(SecretResolutionError::Empty(reference.source_description())); + } + Ok(SecretValue::new(value)) + } +} + +#[async_trait] +impl SecretSourceResolver for SecretResolver { + async fn resolve_secret( + &self, + reference: &SecretReference, + cancellation: &CancellationToken, + ) -> Result { + self.resolve(reference, cancellation) + .await + .map_err(|error| error.to_string()) + } +} + +fn secret_utf8( + bytes: Vec, + reference: &SecretReference, +) -> Result { + String::from_utf8(bytes).map_err(|error| { + let mut bytes = error.into_bytes(); + bytes.fill(0); + SecretResolutionError::InvalidUtf8(reference.source_description()) + }) +} + +fn strip_one_line_ending(mut value: String) -> String { + if value.ends_with("\r\n") { + value.truncate(value.len() - 2); + } else if value.ends_with('\n') { + value.truncate(value.len() - 1); + } + value +} + +#[derive(Debug, Error)] +pub enum SecretResolutionError { + #[error("{0}")] + Policy(#[from] PolicyError), + #[error("{0} is unavailable")] + Unavailable(String), + #[error("{0} resolved to an empty value")] + Empty(String), + #[error("{0} did not contain UTF-8")] + InvalidUtf8(String), + #[error("{reference} exceeded the {limit_bytes}-byte output limit")] + OutputLimit { reference: String, limit_bytes: u64 }, + #[error("{reference} exited unsuccessfully with code {code:?}")] + ProcessExit { + reference: String, + code: Option, + }, + #[error("secret process failed: {0}")] + Process(#[from] ProcessRunError), +} + +#[cfg(test)] +mod tests { + use std::fs; + + use agentctl_core::dsl::{PolicyDefinition, SecretProcessReference}; + use tempfile::tempdir; + + use super::*; + + fn policy(root: &std::path::Path) -> PolicyEngine { + PolicyEngine::new( + PolicyDefinition { + workspace_root: root.display().to_string(), + secret_file_roots: vec!["secrets".to_owned()], + secret_process_allowlist: vec!["sh".to_owned()], + ..PolicyDefinition::default() + }, + root, + ) + .expect("secret policy") + } + + #[tokio::test] + async fn mounted_file_is_bounded_and_strips_one_line_ending() { + let directory = tempdir().expect("tempdir"); + fs::create_dir(directory.path().join("secrets")).expect("secret root"); + fs::write(directory.path().join("secrets/token"), b"file-secret\r\n").expect("secret file"); + let resolver = SecretResolver::restricted(policy(directory.path())); + let value = resolver + .resolve( + &SecretReference::File { + file: "secrets/token".to_owned(), + }, + &CancellationToken::new(), + ) + .await + .expect("resolved file"); + assert_eq!(value.expose(), "file-secret"); + + assert!(matches!( + resolver + .resolve( + &SecretReference::File { + file: "secrets/missing".to_owned() + }, + &CancellationToken::new() + ) + .await, + Err(SecretResolutionError::Policy( + PolicyError::SecretFileDenied(_) + )) + )); + + fs::write( + directory.path().join("secrets/oversized"), + vec![b'x'; usize::try_from(MAX_SECRET_FILE_BYTES + 1).expect("size")], + ) + .expect("oversized file"); + assert!(matches!( + resolver + .resolve( + &SecretReference::File { + file: "secrets/oversized".to_owned() + }, + &CancellationToken::new() + ) + .await, + Err(SecretResolutionError::OutputLimit { .. }) + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn mounted_file_rejects_a_symlink_escape() { + use std::os::unix::fs::symlink; + + let directory = tempdir().expect("tempdir"); + let outside = tempdir().expect("outside"); + fs::create_dir(directory.path().join("secrets")).expect("secret root"); + fs::write(outside.path().join("token"), b"outside-secret").expect("outside file"); + symlink( + outside.path().join("token"), + directory.path().join("secrets/token"), + ) + .expect("symlink"); + let resolver = SecretResolver::restricted(policy(directory.path())); + assert!(matches!( + resolver + .resolve( + &SecretReference::File { + file: "secrets/token".to_owned() + }, + &CancellationToken::new() + ) + .await, + Err(SecretResolutionError::Policy( + PolicyError::SecretFileDenied(_) + )) + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn process_provider_is_allowlisted_bounded_and_timed_out() { + let directory = tempdir().expect("tempdir"); + fs::create_dir(directory.path().join("secrets")).expect("secret root"); + let resolver = SecretResolver::restricted(policy(directory.path())); + let value = resolver + .resolve( + &SecretReference::Process { + process: SecretProcessReference { + command: "/bin/sh".to_owned(), + args: vec!["-c".to_owned(), "printf process-secret".to_owned()], + timeout_seconds: 5, + output_limit_bytes: 64, + }, + }, + &CancellationToken::new(), + ) + .await + .expect("resolved process"); + assert_eq!(value.expose(), "process-secret"); + + let timeout = resolver + .resolve( + &SecretReference::Process { + process: SecretProcessReference { + command: "/bin/sh".to_owned(), + args: vec!["-c".to_owned(), "sleep 2".to_owned()], + timeout_seconds: 1, + output_limit_bytes: 64, + }, + }, + &CancellationToken::new(), + ) + .await + .expect_err("process timeout"); + assert!(matches!( + timeout, + SecretResolutionError::Process(ProcessRunError::Timeout { seconds: 1 }) + )); + + let output_limit = resolver + .resolve( + &SecretReference::Process { + process: SecretProcessReference { + command: "/bin/sh".to_owned(), + args: vec!["-c".to_owned(), "printf 123456789".to_owned()], + timeout_seconds: 5, + output_limit_bytes: 4, + }, + }, + &CancellationToken::new(), + ) + .await + .expect_err("process output limit"); + assert!(matches!( + output_limit, + SecretResolutionError::Process(ProcessRunError::OutputLimitExceeded { .. }) + )); + + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let cancelled = resolver + .resolve( + &SecretReference::Process { + process: SecretProcessReference { + command: "/bin/sh".to_owned(), + args: vec!["-c".to_owned(), "sleep 2".to_owned()], + timeout_seconds: 5, + output_limit_bytes: 64, + }, + }, + &cancellation, + ) + .await + .expect_err("process cancellation"); + assert!(matches!( + cancelled, + SecretResolutionError::Process(ProcessRunError::Cancelled) + )); + + let denied = SecretResolver::restricted( + PolicyEngine::new(PolicyDefinition::default(), directory.path()) + .expect("denied policy"), + ) + .resolve( + &SecretReference::Process { + process: SecretProcessReference { + command: "/bin/sh".to_owned(), + args: Vec::new(), + timeout_seconds: 1, + output_limit_bytes: 64, + }, + }, + &CancellationToken::new(), + ) + .await; + assert!(matches!( + denied, + Err(SecretResolutionError::Policy( + PolicyError::SecretProcessDenied(_) + )) + )); + } +} diff --git a/docs/CONTAINER.md b/docs/CONTAINER.md index 18163b4..8c7f7ec 100644 --- a/docs/CONTAINER.md +++ b/docs/CONTAINER.md @@ -24,7 +24,16 @@ The `Containerfile` combines the secret with public roots on a tmpfs mount for t | `/state` | writable SQLite database, CAS blobs, and durable recovery state | | `/artifacts` | writable declared workflow output/export surface | -Pass workflow values with repeated `--input KEY=VALUE`, `--inputs-file`, or `--inputs` JSON. Prefer files for large or sensitive non-provider inputs. Provider credentials are environment references only; never put a key in CLI arguments, YAML, an image layer, or an ordinary input value. Before a bind-mount run, provision `/state` and `/artifacts` host directories so UID/GID 65532 can write them. Successful bounded workflow files are copied into `/state/artifacts/sha256`; `/artifacts` remains the convenient CI collection surface. Durable state may contain prompts, outputs, and artifact bytes; protect it like a sensitive build artifact. +Pass workflow values with repeated `--input KEY=VALUE`, `--inputs-file`, or +`--inputs` JSON. Prefer files for large or sensitive non-provider inputs. +Provider credentials may reference a forwarded environment name or a read-only +mounted file under an explicit `secretFileRoots` policy. Never put a key in CLI +arguments, YAML, an image layer, or an ordinary input value. Before a bind-mount +run, provision `/state` and `/artifacts` host directories so UID/GID 65532 can +write them. Successful bounded workflow files are copied into +`/state/artifacts/sha256`; `/artifacts` remains the convenient CI collection +surface. Durable state may contain prompts, outputs, and artifact bytes; +protect it like a sensitive build artifact. The image emits exactly one versioned JSON result on stdout with `--output json`; failures emit one versioned JSON error on stderr. The document includes exit status semantics, run/trace IDs, final state, and declared outputs. Progress is not mixed into stdout. Persist `/state` for later `inspect`, approval resolution, `resume`, `replay`, or `repair`. @@ -46,6 +55,26 @@ docker run --rm --read-only --user 65532:65532 \ The value form `--env OPENAI_API_KEY` forwards an already protected host variable without placing its value in the command. The credential-free container acceptance uses the same command with the fake provider and without that environment variable. +For a container-native secret file, configure +`credential: { file: /run/secrets/openai }` and +`secretFileRoots: [/run/secrets]`, then replace the environment forwarding with +a read-only mount: + +```console +docker run --rm --read-only --user 65532:65532 \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --mount type=bind,src="$PWD/config",dst=/config,readonly \ + --mount type=bind,src="$PWD/workspace",dst=/workspace,readonly \ + --mount type=bind,src="$PWD/state",dst=/state \ + --mount type=bind,src="$PWD/openai.key",dst=/run/secrets/openai,readonly \ + ghcr.io/OWNER/agentctl:0.2.0 \ + run /config/workflow.yaml --workspace /workspace --db /state/runtime.db \ + --output json --color never +``` + +The file is read at bounded credential preflight and its value is never copied +to the state mount. See [Secret references](guides/SECRET_REFERENCES.md). + For selective repair, mount the corrected workflow under `/config` and keep the source database plus its `/state/artifacts` CAS under `/state`. The original workspace output can be absent after successful ingestion. Plan without forwarding provider credentials: ```console @@ -201,7 +230,13 @@ The surrounding Harness stage must publish `/harness/.agentctl-state` and `/harn ### Kubernetes Job or CronJob -Use ConfigMaps for reviewed configuration, a PVC for `/state` when recovery across Pods matters, a PVC or artifact uploader for `/artifacts`, and a Secret environment reference for credentials. The container security context should set `runAsNonRoot`, UID/GID 65532, no privilege escalation, dropped capabilities, and a read-only root filesystem. A CronJob should normally set `concurrencyPolicy: Forbid`; see [Operations](OPERATIONS.md). +Use ConfigMaps for reviewed configuration, a PVC for `/state` when recovery +across Pods matters, a PVC or artifact uploader for `/artifacts`, and either a +Secret environment reference or a projected read-only Secret volume for +credentials. The container security context should set `runAsNonRoot`, UID/GID +65532, no privilege escalation, dropped capabilities, and a read-only root +filesystem. A CronJob should normally set `concurrencyPolicy: Forbid`; see +[Operations](OPERATIONS.md). ```yaml apiVersion: batch/v1 diff --git a/docs/DSL.md b/docs/DSL.md index 081f6b2..efd1f8f 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -8,7 +8,12 @@ Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }} Task output is JSON. Built-in actions own an object contract, agents can declare provider-enforced `structuredOutput`, and a task can override the complete contract with `outputSchema`. The compiler validates schemas; the runtime validates completed and selectively reused values. -Providers, action environments, and protocol headers use `{ env: NAME }` secret references. Secret names are validated and values never become the workflow document. +Providers, action environments, and protocol headers use secret references: +`{ env: NAME }`, `{ file: PATH }`, or a bounded `{ process: ... }` reference. +File references require `policy.secretFileRoots`; process references require +`policy.secretProcessAllowlist`. Existing environment references remain +compatible. Resolved values never become the workflow document, effect value, +trace, or inspection output. See [Secret references](guides/SECRET_REFERENCES.md). `policy.workspaceRoot` is the default boundary for relative file paths. Each `writableRoots` entry may be workspace-relative or an explicit absolute mount such as `/artifacts`. Ordinary reads remain workspace-confined. After a successful authorized mutation, the runtime may read that exact output through its writable-root boundary to ingest the bounded regular file into durable CAS; this does not grant tasks general read access to the external root. diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index 4d48adb..b800e93 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -41,6 +41,11 @@ When diagnosing a failure, correlate the final envelope's run and trace IDs with Sensitive field names and registered secret values are redacted before trace attributes leave the runtime. Provider response content is not printed by the live smoke. Operators must still treat trace backends and the local database as sensitive because prompts, file content, tool output, and remote artifacts may contain confidential non-secret data. -Keep provider credentials in environment references, never workflow inputs or command arguments. Apply access control and retention to the database, collected artifacts, CI logs, and trace backend. Before sharing diagnostics, remove credentials, prompt content, file content, remote payloads, and identifying metadata; a run ID alone is sufficient for local correlation. +Keep provider credentials in typed environment, mounted-file, or policy-gated +process references, never workflow inputs or command arguments. Apply access +control and retention to the database, collected artifacts, CI logs, and trace +backend. Before sharing diagnostics, remove credentials, prompt content, file +content, remote payloads, and identifying metadata; a run ID alone is +sufficient for local correlation. See [CLI output and exit codes](reference/CLI_OUTPUT.md), [local operation](guides/LOCAL_OPERATION.md), and [runtime database and migrations](reference/DATABASE.md) for the complete operating contract. diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index ebad1c9..0f2a81f 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -10,7 +10,15 @@ The core defines provider-neutral messages, text/reasoning/tool content, strict | `anthropic` | Messages API | native content/tool/thinking blocks, structured output instruction, usage and stop mapping | `ANTHROPIC_API_KEY` | | `google` | Gemini `generateContent` | native contents/function declarations/calls/results, thought-signature continuation, response schema, token usage | `GEMINI_API_KEY` | -Endpoints must pass the workflow network allowlist. Redirects are disabled. Credentials and configured headers are resolved from environment references only when building an adapter; standard authentication headers override custom headers. Successful/error response JSON keys and values plus provider request IDs are scrubbed of configured secrets before parsing or persistence. Calls honor timeout and cancellation. +Endpoints must pass the workflow network allowlist. Redirects are disabled. +Credentials and configured headers accept environment, mounted-file, or +policy-gated process references. Provider credentials in the fresh execution +closure are preflighted before a new run record or effect; custom headers +resolve while building a required adapter. +Standard authentication headers override custom headers. Successful/error +response JSON keys and values plus provider request IDs are scrubbed of +configured secrets before parsing or persistence. Calls honor timeout and +cancellation. See [Secret references](guides/SECRET_REFERENCES.md). `agentctl providers inspect ` reports declared capabilities without calling a service. OpenAI has the broadest mock request/response/tool/usage/error coverage. Azure OpenAI, Anthropic, and Google have native mapping and focused mock-protocol coverage at the maturity shown below; normal tests have no credentials. Live provider workflow examples end in `-live.yaml` and are opt-in. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 6b7d623..4f1ca21 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -3,7 +3,16 @@ ## Controls - Workflow parsing is strict, bounded to 1 MiB, source-aware, and has no executable expression language. -- Environment-backed primary credentials are resolved immediately before provider dispatch; custom header references are resolved while constructing the adapter, before a run or database is created. There are no API-key flags. Provider/protocol response JSON keys and values, provider request IDs, errors, subprocess output, and traces redact every known configured secret value before persistence or output. +- Provider credentials in the fresh execution closure are preflighted before a + new run record or effect is created; custom header references are resolved + while constructing a required adapter; action environment references are + resolved at the task boundary. References may use an environment variable, a + bounded canonical file under `secretFileRoots`, or a direct bounded process + under `secretProcessAllowlist`. Values use zeroizing memory wrappers and never + enter ordinary persisted state. There are no API-key flags. + Provider/protocol response JSON keys and values, provider request IDs, errors, + subprocess output, and traces redact every known configured secret value + before persistence or output. - Canonical read/write roots reject `..` and symlink escape. Writes use temporary files and rename. - Processes require an allowed executable basename, direct argv, cleared environment, selected variables, validated output/timeout bounds, concurrent stdout/stderr draining, and cancellation. Output-limit, timeout, and cancellation paths terminate and reap the child; diagnostics are bounded and omit captured output when secret environment values are present. - Network destinations require an exact/wildcard host grant. Provider and protocol clients disable redirects and use rustls. @@ -18,7 +27,14 @@ ## Limitations -Path and executable allowlists are not a sandbox. A permitted program can access anything the operating-system identity can access. Host allowlists do not defend against every DNS rebinding, proxy, local-service, or compromised endpoint scenario; use network isolation for hostile workflows. SHA-256 integrity establishes sameness, not author identity. State encryption is application-level selected-field protection, not full-database encryption, access control, or a secret store. +Path and executable allowlists are not a sandbox. A permitted program, including +a secret helper, can access anything the operating-system identity can access. +Redaction cannot prevent an authorized recipient from transforming a secret +before exfiltration. Host allowlists do not defend against every DNS rebinding, +proxy, local-service, or compromised endpoint scenario; use network isolation +for hostile workflows. SHA-256 integrity establishes sameness, not author +identity. State encryption is application-level selected-field protection, not +full-database encryption, access control, or a secret store. Prompts, file content, model output, remote artifacts, and tool output may be confidential or malicious. Treat them as data, validate before mutation, minimize trace export, and isolate untrusted automation. Workflow, input, pack, direct-read, existing-write-target, and instruction files are capped at 1 MiB. Approval is a decision point, not proof that an operation is safe. At-most-once recovery may leave an uncertain external outcome for human reconciliation. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 8498499..a93dbe9 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -2,7 +2,12 @@ ## Assets and boundaries -Assets are workspace files, content-addressed artifact bytes, allowed environment secrets, provider accounts, external systems reached by tools, workflow history, prompts/results, approvals, and the integrity of deterministic scheduling. Boundaries are the YAML/pack parser, filesystem/process/network executors, provider APIs, MCP servers, A2A peers, SQLite and its sibling artifact root, trace exporters, and dependencies. +Assets are workspace files, content-addressed artifact bytes, referenced +environment/file/process secrets, provider accounts, external systems reached +by tools, workflow history, prompts/results, approvals, and the integrity of +deterministic scheduling. Boundaries are the YAML/pack parser, secret resolver, +filesystem/process/network executors, provider APIs, MCP servers, A2A peers, +SQLite and its sibling artifact root, trace exporters, and dependencies. The local operator and reviewed binary are trusted. Workflow authors are only as trusted as policy grants. Models, file content, remote descriptions/results, pack content without independent provenance, and all network peers are untrusted. The host OS, CA store, and Rust dependency supply chain are assumed but monitored dependencies. @@ -10,7 +15,7 @@ The local operator and reviewed binary are trusted. Workflow authors are only as | --- | --- | --- | | Malicious YAML/template causes code execution or resource exhaustion | strict fields, constrained paths/equality, 1 MiB bound, fuzzing | deeply nested valid data remains bounded mainly by parser behavior | | Path traversal or symlink escape | canonical roots and focused tests | TOCTOU is possible if another process swaps paths; isolate hostile workspaces | -| Secret exfiltration through CLI/log/database/trace | env references, no key flags, allowlists, redaction, secret scan | authorized tools can deliberately transmit permitted data | +| Secret exfiltration through CLI/log/database/trace | typed references, canonical file roots, bounded allowlisted process helpers, zeroizing values, no key flags, redaction, raw-database tests, secret scan | authorized recipients can deliberately transmit or transform permitted data | | SQLite disclosure reveals confidential run content | optional AES-256-GCM field envelopes, external key reference, authenticated context, fail-closed triggers, transactional rotation | metadata and artifact bytes remain visible; unencrypted and pre-migration backups remain sensitive | | Command injection | direct argv, no shell, cleared env, executable allowlist | an allowed executable may interpret malicious arguments | | SSRF/redirect bypass | URL parse, host allowlist, disabled redirects, tests | DNS/proxy behavior needs external network containment for hostile inputs | diff --git a/docs/adr/0006-schedulable-runtime-and-noninteractive-contract.md b/docs/adr/0006-schedulable-runtime-and-noninteractive-contract.md index e573d9f..682043a 100644 --- a/docs/adr/0006-schedulable-runtime-and-noninteractive-contract.md +++ b/docs/adr/0006-schedulable-runtime-and-noninteractive-contract.md @@ -8,7 +8,11 @@ Status: accepted Non-interactive execution never prompts or auto-approves. The default approval behavior persists the request, pauses the run, emits run/trace correlation, and exits `3`. An operator resolves the approval and invokes `resume`. `deny_approval` and `fail` are stricter explicit modes. -Machine output is one `agentctl.dev/cli/v1` final envelope. Inputs come from JSON, an input file, or repeated `KEY=VALUE` arguments; provider secrets remain environment references. Separate runs can share a SQLite database, but external schedulers must prevent overlapping effects when the target resource requires serialization. +Machine output is one `agentctl.dev/cli/v1` final envelope. Inputs come from +JSON, an input file, or repeated `KEY=VALUE` arguments; provider secrets remain +typed references rather than values. Separate runs can share a SQLite database, +but external schedulers must prevent overlapping effects when the target +resource requires serialization. ## Consequences diff --git a/docs/development/ADD_PROVIDER.md b/docs/development/ADD_PROVIDER.md index 97924fd..8f0e75a 100644 --- a/docs/development/ADD_PROVIDER.md +++ b/docs/development/ADD_PROVIDER.md @@ -8,7 +8,10 @@ Declare support for text, structured output, tools, reasoning, continuation, cac ## Authentication and network boundary -Use a workflow environment reference. Resolve credentials only at the adapter boundary, never from a CLI key flag. Enforce the reviewed endpoint host, disable redirects, use rustls, and define whether an endpoint override is permitted. +Use the core secret-reference contract and runtime resolver. Resolve credentials +only at the adapter boundary, never from a CLI key flag. Enforce the reviewed +endpoint host, disable redirects, use rustls, and define whether an endpoint +override is permitted. ## Native request mapping diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 1637799..9c93d58 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -73,13 +73,13 @@ cargo xtask acceptance-container | Reconciliation | immutable transition matrix, schema/tool/hook/policy, repair and resume tests | full composite rerun pending | verified | | Terminal retry | runtime/store identity, roots, acknowledgements, reconciliation, lineage, source immutability, and replay tests passed | packaged CLI scenario 30 and the 12-stage verification gate passed | verified | | Sensitive-state encryption | authenticated context, wrong-key, tamper, inventory, stale-writer trigger, rollback, rotation, checkpoint, and retained-schema tests passed | packaged CLI scenario 31 and the 12-stage verification gate passed | verified | +| Secret references | environment compatibility, file bounds/missing/symlink containment, process allowlist/timeout/output/cancellation, zeroizing values, adapter redaction, and raw-database absence tests passed | packaged CLI scenario 32 and the 12-stage verification gate passed | verified | | Parallel/dynamic workflows | pending | pending | open | | Conditions/loops/sub-workflows | pending | pending | open | | Compensation/handoffs/streaming | pending | pending | open | | MCP/A2A resilience | pending | pending | open | | Packs/trust/extensions | pending | pending | open | | Semantic memory | pending | pending | open | -| Encryption/secrets | pending | pending | open | | Network/isolation/budgets | pending | pending | open | | Container/cross-platform | baseline defect recorded | pending | in progress | | OpenAI live matrix | retained selective-repair evidence only | pending | open | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index 39efe36..99ea3d7 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -39,7 +39,7 @@ complete, every entry must have exactly one final disposition: | EFX-001 | Effect reconciliation | verified | implemented | | RET-001 | Terminal-run retry | verified | implemented | | ENC-001 | Sensitive-state encryption | verified | implemented | -| SEC-001 | Secret providers | open | implemented | +| SEC-001 | Secret providers | verified | implemented | | NET-001 | Network policy | open | implemented | | ISO-001 | Process isolation | open | redesigned | | BUD-001 | Resource and cost budgets | open | implemented | @@ -508,23 +508,30 @@ complete, every entry must have exactly one final disposition: ### SEC-001: Stable secret-reference providers -- Current behavior: environment references are supported; mounted file and - policy-gated command providers are absent. -- User impact: container-native secret files require wrapper scripts. -- Security or durability impact: wrappers may place resolved values in ordinary - inputs or arguments. +- Current behavior: environment, bounded mounted-file, and policy-gated direct + process references work across provider credentials, provider/protocol + headers, and action environments. +- User impact: container-native secret files and reviewed credential helpers + work without wrapper scripts or secret-valued workflow inputs. +- Security or durability impact: resolved values stay in zeroizing memory, + while effect records retain only source descriptions and value digests. - Product decision: version secret references for environment, bounded mounted file, and optional direct process provider. Resolved values never persist. -- Required implementation: policy allowlists, path containment, process argv, - timeout/output bound, redaction registration, and lifecycle zeroization where - practical. +- Required implementation: canonical file-root containment, dedicated process + allowlists, direct argv, cleared environments, process groups, + timeout/output/cancellation bounds, redaction registration, and lifecycle + zeroization. - Migration impact: existing `{env: NAME}` remains valid. - Tests: missing/oversized/symlink files, denied commands, timeout, redaction, and database/trace absence. -- Examples: environment and mounted-file container secrets. +- Examples: environment and read-only mounted-file container contracts in the + secret-reference guide and container documentation. - Live evidence: OpenAI credential remains environment-only for task evidence. - Documentation: secret reference types and threat model. -- Final disposition: pending implementation evidence. +- Final disposition: implemented and verified by DSL compatibility and policy + tests; missing, oversized, and symlink-escape file tests; denied, timed-out, + output-limited, and cancelled process tests; provider adapter redaction; raw + SQLite absence checks; and packaged CLI acceptance scenario 32. ### NET-001: Network destination enforcement diff --git a/docs/guides/CI_CD.md b/docs/guides/CI_CD.md index d85f964..903bf95 100644 --- a/docs/guides/CI_CD.md +++ b/docs/guides/CI_CD.md @@ -42,7 +42,11 @@ Their current evidence level is documentation or syntax review unless stated oth ## Inputs and structured output -Mount an ordinary JSON file under `/config` and pass `--inputs-file /config/inputs.json`, or use repeated non-secret `--input KEY=VALUE`. Provider credentials must be environment references. Capture stdout as one `agentctl.dev/cli/v1` JSON envelope and archive declared files from `/artifacts`. +Mount an ordinary JSON file under `/config` and pass `--inputs-file +/config/inputs.json`, or use repeated non-secret `--input KEY=VALUE`. Provider +credentials must be typed environment or mounted-file references. Capture +stdout as one `agentctl.dev/cli/v1` JSON envelope and archive declared files +from `/artifacts`. ## Approvals in pipelines @@ -71,7 +75,7 @@ Do not set a pipeline retry policy that blindly repeats exit `5`, `6`, or `130`. - Run as non-root with a read-only root filesystem. - Drop capabilities and deny unneeded egress. - Mount the workspace read-only unless a reviewed write is required. -- Inject secrets by environment reference and never echo them. +- Inject secrets by environment reference or read-only mounted file and never echo them. - Treat remote content and model output as untrusted. - Retain state for approval or recovery, then delete it under policy. - Set the external platform's overlap and timeout controls. diff --git a/docs/guides/SECRET_REFERENCES.md b/docs/guides/SECRET_REFERENCES.md new file mode 100644 index 0000000..060128b --- /dev/null +++ b/docs/guides/SECRET_REFERENCES.md @@ -0,0 +1,140 @@ +# Secret references + +Workflow YAML stores references, never resolved secret values. `agentctl` +supports three reference forms: + +```yaml +credential: { env: OPENAI_API_KEY } +credential: { file: /run/secrets/openai } +credential: + process: + command: /usr/local/bin/secret-helper + args: [read, openai] + timeoutSeconds: 5 + outputLimitBytes: 16384 +``` + +Existing `{ env: NAME }` documents remain compatible. Inline secret strings, +secret-valued CLI arguments, and automatic external secret-manager adapters are +not supported. + +## Environment references + +An environment reference must use a valid variable name. Primary provider +credentials keep the established convention of `OPENAI_API_KEY`, +`AZURE_OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `GEMINI_API_KEY` and do not +require a duplicate `policy.environmentAllowlist` entry. Environment +references used for custom HTTP headers or action environment values must be +listed in `environmentAllowlist`. + +Only providers reached by an agent task in the execution closure are +credential-preflighted. Resolution occurs before a fresh run record or effect +is created. Resume, retry, and repair derive that closure from durable task +state or the accepted plan, so a reused provider task does not require its +credential. An unused declared provider does not make an otherwise +deterministic workflow require a credential. + +## Mounted-file references + +File references require at least one `policy.secretFileRoots` entry: + +```yaml +spec: + providers: + openai: + kind: openai + credential: { file: /run/secrets/openai } + policy: + secretFileRoots: [/run/secrets] + networkAllowlist: [api.openai.com] +``` + +Relative references and roots are resolved from `policy.workspaceRoot`. +Absolute roots support container secret mounts. The resolved target must be an +existing regular file whose canonical path remains inside a canonical allowed +root. A symlink contained by the root is accepted, which supports projected +secret rotation. A symlink escape, directory, missing path, or `..` traversal +is rejected. + +Reads are limited to 64 KiB and must be UTF-8. Exactly one trailing LF or CRLF +is removed to support ordinary secret files. Empty and oversized values fail +closed. + +## Process references + +A process reference is disabled unless its executable basename appears in +`policy.secretProcessAllowlist`: + +```yaml +spec: + policy: + secretProcessAllowlist: [secret-helper] + providers: + openai: + kind: openai + credential: + process: + command: /usr/local/bin/secret-helper + args: [read, openai] + timeoutSeconds: 5 + outputLimitBytes: 16384 +``` + +The helper is invoked directly with the declared argument vector. No shell is +inserted. It starts in the policy workspace with a cleared environment, belongs +to a terminable process group where supported, and receives the run +cancellation token. `timeoutSeconds` defaults to 5 and is limited to 60. +`outputLimitBytes` defaults to 16 KiB and is limited to 64 KiB. Standard error +is bounded separately and never becomes the secret or an error message. +Nonzero exit, timeout, cancellation, non-UTF-8 output, empty output, and output +overflow all fail closed. + +Process references are useful for an already installed, reviewed credential +helper. They do not turn workflow policy into an operating-system sandbox. An +allowed helper runs with the `agentctl` process identity. + +## Container-mounted secret + +Mount the secret read-only and grant only its parent directory: + +```console +docker run --rm --read-only --user 65532:65532 \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --mount type=bind,src="$PWD/config",dst=/config,readonly \ + --mount type=bind,src="$PWD/workspace",dst=/workspace,readonly \ + --mount type=bind,src="$PWD/state",dst=/state \ + --mount type=bind,src="$PWD/openai.key",dst=/run/secrets/openai,readonly \ + ghcr.io/OWNER/agentctl:0.2.0 \ + run /config/workflow.yaml --workspace /workspace \ + --db /state/runtime.db --output json --color never +``` + +Kubernetes projected Secrets and Docker or Compose secrets can use the same +`/run/secrets` workflow contract. Do not copy a secret into the image or state +mount. + +## Resolution, redaction, and persistence + +Provider credentials are preflighted for reachable provider tasks, protocol +headers are resolved while the execution registry is built, and action +environment references are resolved at their task boundary. Resolved values +are held in zeroizing in-memory wrappers. File and process resolution uses the +same cancellation token as the run. + +Only the safe source description and a SHA-256 value digest enter an action +effect record. Provider/protocol responses and subprocess output redact every +resolved value before output, tracing, or persistence. Inspection never +returns a resolved value: + +```console +agentctl auth check workflow.yaml --output json +agentctl providers inspect workflow.yaml --output json +``` + +`auth check` tests environment presence and file availability without reading +their values. It deliberately does not execute a process reference. + +Redaction protects accidental echoes, not deliberate exfiltration. Any +authorized provider, protocol peer, helper, or subprocess that receives a +secret can transform or transmit it. Use reviewed workflows, least-privilege +credentials, container isolation, and egress controls. diff --git a/docs/guides/TROUBLESHOOTING.md b/docs/guides/TROUBLESHOOTING.md index 746368f..cf64f89 100644 --- a/docs/guides/TROUBLESHOOTING.md +++ b/docs/guides/TROUBLESHOOTING.md @@ -37,9 +37,11 @@ agentctl schema --write /tmp/workflow.schema.json --output json --color never ## Provider authentication failure -**Symptom:** Exit `6` reports a missing environment reference or authentication response. +**Symptom:** Exit `6` reports an unavailable secret reference or authentication response. -**Likely cause:** The workflow names a credential environment variable that is absent or the provider rejected it. +**Likely cause:** The workflow names an absent environment value, unavailable +or denied file/process source, or the provider rejected the resolved +credential. **Diagnose:** @@ -48,9 +50,13 @@ agentctl auth check workflow.yaml --output json --color never agentctl providers inspect workflow.yaml --output json --color never ``` -**Expected evidence:** The environment variable name and provider capability, never the secret value. +**Expected evidence:** The safe source description and provider capability, +never the secret value. Process references report `unchecked` and are not +executed by diagnostics. -**Resolve:** Inject the named secret through the shell, scheduler, or CI secret facility. Do not add a key to YAML or a command argument. +**Resolve:** Inject the named environment value, mount the file under an allowed +root, or repair the process policy/helper. Do not add a key to YAML or a command +argument. ## Provider capability mismatch diff --git a/docs/policies.md b/docs/policies.md index 2f10aa5..16291d4 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -1,8 +1,20 @@ # Policies and approvals -Policy is evaluated by the runtime, never by a model. A policy defines a canonical workspace root, writable roots, allowed environment names, network host patterns, process basenames, providers, tool allow/deny lists, approval mode, and non-interactive behavior. +Policy is evaluated by the runtime, never by a model. A policy defines a +canonical workspace root, writable roots, allowed environment names, secret +file roots, ordinary and secret-helper process basenames, network host +patterns, providers, tool allow/deny lists, approval mode, and non-interactive +behavior. -Read paths must canonicalize under the workspace. Write paths canonicalize the nearest existing parent and must remain under a writable root. Parent traversal and symlink escape fail. Network rules match an exact hostname or `*.suffix` subdomains; suffix lookalikes and the wildcard apex do not match. HTTP redirects are disabled. Process allowlisting checks the executable basename and then launches direct argv with a cleared environment. +Read paths must canonicalize under the workspace. Write paths canonicalize the +nearest existing parent and must remain under a writable root. Secret files +must be existing regular files canonically contained by `secretFileRoots`. +Parent traversal and symlink escape fail. Network rules match an exact hostname +or `*.suffix` subdomains; suffix lookalikes and the wildcard apex do not match. +HTTP redirects are disabled. Process allowlisting checks the executable +basename and then launches direct argv with a cleared environment. Secret +helpers use their separate `secretProcessAllowlist` and stricter 60-second, +64-KiB maximums. See [Secret references](guides/SECRET_REFERENCES.md). Tool visibility, tool/capability authorization, resource checks, effect risk, and approval are distinct decisions. `never`, `mutations`, `high_risk`, and `always` are available approval modes. A tool may say `never`, `policy`, or `always`. The default non-interactive behavior is a durable pause and exit code `3`; explicit `deny_approval` and `fail` modes fail closed. Non-interactive execution never prompts or auto-approves. diff --git a/docs/reference/ENVIRONMENT_AND_PATHS.md b/docs/reference/ENVIRONMENT_AND_PATHS.md index fddaf1f..372363e 100644 --- a/docs/reference/ENVIRONMENT_AND_PATHS.md +++ b/docs/reference/ENVIRONMENT_AND_PATHS.md @@ -10,7 +10,12 @@ | `anthropic` | `ANTHROPIC_API_KEY` | The workflow dispatches an Anthropic request. | | `google` | `GEMINI_API_KEY` | The workflow dispatches a Google request. | -These names are defaults used by repository examples. A workflow can name another valid environment reference. Policy must allow the name. Values never belong in YAML, CLI arguments, ordinary inputs, logs, or committed fixtures. +These names are defaults used by repository examples. A workflow can name +another valid environment reference or use a mounted-file or policy-gated +process reference. Primary provider credential environment names do not require +a duplicate environment allowlist entry; custom headers and action environment +values do. Values never belong in YAML, CLI arguments, ordinary inputs, logs, +or committed fixtures. See [Secret references](../guides/SECRET_REFERENCES.md). ## State-encryption keys @@ -44,6 +49,7 @@ Normal `cargo xtask docs-verify`, `cargo xtask verify`, and `cargo xtask accepta | `/workspace` | normally read-only workspace | | `/state` | writable SQLite and content-addressed durable state | | `/artifacts` | writable workflow output/export mount | +| `/run/secrets` | optional read-only mounted secret files granted through `secretFileRoots` | | `/tmp` | small runtime tmpfs when the root filesystem is read-only | State and artifacts must be writable by UID/GID 65532 in the production image. diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index c9343bf..b6b0298 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -96,7 +96,12 @@ An exact template preserves objects, arrays, booleans, numbers, strings, and nul ## Secret references -Provider credentials, action environment values, and protocol headers use `{ env: NAME }`. The environment name is stored in the workflow, but the value is resolved only at the adapter boundary and must be allowed by policy. +Provider credentials, action environment values, and protocol headers use +`{ env: NAME }`, `{ file: PATH }`, or a bounded `{ process: ... }` reference. +The source description is stored in the workflow, but the value is resolved +only at the execution boundary. File and process sources require explicit +`secretFileRoots` or `secretProcessAllowlist` policy. See +[Secret references](../guides/SECRET_REFERENCES.md). ## Example and validation @@ -108,4 +113,6 @@ agentctl plan examples/v1/dataflow.yaml agentctl run examples/v1/dataflow.yaml --db /tmp/dataflow.db --output json --color never ``` -Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Policies](../policies.md), [Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). +Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Secret +references](../guides/SECRET_REFERENCES.md), [Policies](../policies.md), +[Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). diff --git a/docs/use-cases/CI_QUALITY_GATE.md b/docs/use-cases/CI_QUALITY_GATE.md index 730e200..b6da063 100644 --- a/docs/use-cases/CI_QUALITY_GATE.md +++ b/docs/use-cases/CI_QUALITY_GATE.md @@ -25,7 +25,9 @@ The default exits `0` with verdict `pass`. Run with `--input checksPassed=false` ## State and security -Use ordinary typed inputs for non-secret gate evidence. Inject provider secrets only by environment reference. Archive the database on failure only when its potentially confidential content is protected. +Use ordinary typed inputs for non-secret gate evidence. Inject provider secrets +only through typed environment or mounted-file references. Archive the database +on failure only when its potentially confidential content is protected. ## Current limitation diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 07dd989..7bbb570 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -56,6 +56,7 @@ dependencies = [ "thiserror", "tokio-util", "url", + "zeroize", ] [[package]] diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index dd5fe65..6925fc7 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -109,6 +109,8 @@ "environmentAllowlist": [], "networkAllowlist": [], "processAllowlist": [], + "secretFileRoots": [], + "secretProcessAllowlist": [], "providers": [], "toolsAllow": [], "toolsDeny": [], @@ -215,15 +217,74 @@ ] }, "SecretReference": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "env" + ] + }, + { + "type": "object", + "properties": { + "file": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "file" + ] + }, + { + "type": "object", + "properties": { + "process": { + "$ref": "#/$defs/SecretProcessReference" + } + }, + "additionalProperties": false, + "required": [ + "process" + ] + } + ] + }, + "SecretProcessReference": { "type": "object", "properties": { - "env": { + "command": { "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "timeoutSeconds": { + "type": "integer", + "format": "uint64", + "minimum": 0, + "default": 5 + }, + "outputLimitBytes": { + "type": "integer", + "format": "uint64", + "minimum": 0, + "default": 16384 } }, "additionalProperties": false, "required": [ - "env" + "command" ] }, "AgentDefinition": { @@ -728,6 +789,20 @@ }, "default": [] }, + "secretFileRoots": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "secretProcessAllowlist": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, "providers": { "type": "array", "items": { diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 925b816..6d30062 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 31; +const ACCEPTANCE_SCENARIOS: usize = 32; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1226,6 +1226,92 @@ pub fn run(root: &Path) -> Result<()> { ensure!(!serde_json::to_string(&inventory)?.contains(&key_one)); ensure!(!serde_json::to_string(&rotated)?.contains(&key_two)); + scenario( + 32, + "file and process secret providers are bounded, redacted, and never persisted", + ); + let secret_workspace = workspace.join("secret-providers"); + fs::create_dir_all(secret_workspace.join("secrets"))?; + let secret_marker = "agentctl-mounted-secret-acceptance-marker"; + fs::write( + secret_workspace.join("secrets/token"), + format!("{secret_marker}\n"), + )?; + let secret_workflow = secret_workspace.join("workflow.yaml"); + let binary_path = path(&binary)?; + let secret_document = serde_json::json!({ + "apiVersion": "agentctl.dev/v1alpha1", + "kind": "Workflow", + "metadata": {"name": "secret-providers"}, + "spec": { + "policy": { + "workspaceRoot": ".", + "processAllowlist": ["agentctl"], + "environmentAllowlist": ["FILE_SECRET", "PROCESS_SECRET"], + "secretFileRoots": ["secrets"], + "secretProcessAllowlist": ["agentctl"], + "approval": "never" + }, + "actions": { + "consume": { + "kind": "builtin.shell.exec", + "command": binary_path, + "args": ["version"], + "timeoutSeconds": 5, + "env": { + "FILE_SECRET": {"file": "secrets/token"}, + "PROCESS_SECRET": { + "process": { + "command": binary_path, + "args": ["version"], + "timeoutSeconds": 5, + "outputLimitBytes": 128 + } + } + } + } + }, + "tasks": [{"id": "consume", "uses": "action:consume"}] + } + }); + write( + &secret_workflow, + &serde_json::to_string_pretty(&secret_document)?, + )?; + let secret_db = directory.path().join("secret-providers.db"); + let secret_run = successful_json( + &binary, + &secret_workspace, + &run_args(&secret_workflow, &secret_db, &secret_workspace, &[]), + )?; + let secret_run_id = string_at(&secret_run, "/data/runId")?; + let secret_inspect = inspect(&binary, &secret_workspace, &secret_db, secret_run_id)?; + let secret_inspect_text = serde_json::to_string(&secret_inspect)?; + ensure!(secret_inspect_text.contains("[REDACTED]")); + ensure!(!secret_inspect_text.contains(secret_marker)); + let database_bytes = fs::read(&secret_db)?; + ensure!( + !database_bytes + .windows(secret_marker.len()) + .any(|window| window == secret_marker.as_bytes()) + ); + + fs::remove_file(secret_workspace.join("secrets/token"))?; + let missing_secret_db = directory.path().join("missing-secret-provider.db"); + let missing_secret = json_with_code( + &binary, + &secret_workspace, + &run_args(&secret_workflow, &missing_secret_db, &secret_workspace, &[]), + 4, + )?; + ensure!( + missing_secret + .pointer("/error/message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("secret file")) + ); + ensure!(!serde_json::to_string(&missing_secret)?.contains(secret_marker)); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 29ad89e..e603329 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -691,6 +691,7 @@ fn verify_public_documentation(root: &Path) -> Result<()> { "docs/guides/WORKFLOW_AUTHORING.md", "docs/guides/LOCAL_OPERATION.md", "docs/guides/SENSITIVE_STATE_ENCRYPTION.md", + "docs/guides/SECRET_REFERENCES.md", "docs/guides/TERMINAL_RETRY.md", "docs/guides/repair-a-failed-workflow.md", "docs/guides/LEGACY_RUN_UPGRADE.md", From 9136ed14d8e2c393807809266921b134e9cb71c9 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 20:57:16 +0530 Subject: [PATCH 11/44] feat: add deterministic parallel scheduling --- Cargo.lock | 1 + README.md | 1 + crates/agentctl-cli/src/main.rs | 3 +- crates/agentctl-core/src/compiler.rs | 294 +++ crates/agentctl-core/src/dsl.rs | 14 +- crates/agentctl-runtime/Cargo.toml | 1 + crates/agentctl-runtime/src/lib.rs | 2171 +++++++++++++---- crates/agentctl-store/src/encryption.rs | 4 + crates/agentctl-store/src/lib.rs | 607 ++++- docs/ARCHITECTURE.md | 16 +- docs/COMPATIBILITY.md | 2 +- docs/DSL.md | 6 +- docs/DURABLE_EXECUTION.md | 6 +- docs/LIMITATIONS.md | 4 +- docs/PROVIDERS.md | 2 +- ...005-narrow-v1-scheduling-and-extensions.md | 2 +- .../0008-deterministic-parallel-batches.md | 20 + docs/architecture/DIAGRAMS.md | 12 +- docs/development/REPOSITORY.md | 2 +- docs/execution/DECISIONS.md | 3 +- docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 16 +- docs/execution/RELEASE_AUDIT.md | 2 +- docs/execution/STATUS.md | 8 +- docs/guides/PARALLEL_TASKS.md | 86 + docs/memory.md | 2 +- docs/reference/DATABASE.md | 6 +- docs/reference/YAML.md | 8 +- docs/research/LANDSCAPE.md | 2 +- examples/v1/README.md | 1 + examples/v1/parallel.yaml | 32 + fuzz/Cargo.lock | 1 + schemas/workflow.schema.json | 6 + xtask/src/acceptance.rs | 90 +- xtask/src/main.rs | 1 + 35 files changed, 2858 insertions(+), 575 deletions(-) create mode 100644 docs/adr/0008-deterministic-parallel-batches.md create mode 100644 docs/guides/PARALLEL_TASKS.md create mode 100644 examples/v1/parallel.yaml diff --git a/Cargo.lock b/Cargo.lock index 33a53cf..be57d79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,7 @@ dependencies = [ "agentctl-store", "async-trait", "chrono", + "futures-util", "hex", "jsonschema", "nix", diff --git a/README.md b/README.md index dd699b3..f136926 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md) and [Repair a fai For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). For confidential workflow history, use [Sensitive-state encryption](docs/guides/SENSITIVE_STATE_ENCRYPTION.md). For environment, mounted-file, and policy-gated process credentials, use [Secret references](docs/guides/SECRET_REFERENCES.md). +For bounded independent branches and working-memory conflict rules, use [Deterministic parallel tasks](docs/guides/PARALLEL_TASKS.md). ## Safety boundary diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index 405f2ba..130c0ef 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -647,9 +647,10 @@ async fn execute(cli: Cli) -> Result { &plan, diagnostics, format!( - "plan {}\norder: {}\npredictability: {:?}\nproviders: {}\ntools: {}\neffects: {}", + "plan {}\norder: {}\nmax concurrency: {}\npredictability: {:?}\nproviders: {}\ntools: {}\neffects: {}", plan.plan_digest, plan.order.join(" -> "), + plan.max_concurrency, plan.predictability, plan.requirements .providers diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index 6a8d7ec..dbb58b8 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -26,6 +26,8 @@ pub struct CompiledTask { pub id: String, pub uses: TaskUse, pub needs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub memory_writes: Vec, pub when: Option, pub vars: JsonMap, pub input: JsonMap, @@ -51,6 +53,8 @@ pub struct CompiledPlan { pub workflow_digest: String, pub plan_digest: String, pub order: Vec, + #[serde(default = "default_max_concurrency")] + pub max_concurrency: usize, pub tasks: BTreeMap, pub predictability: PlanPredictability, pub requirements: PlanRequirements, @@ -187,6 +191,15 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), }; input.extend(task.input.clone()); + let memory_writes = task_memory_writes( + workflow, + &task_use, + &input, + &task.memory_writes, + file, + position, + &mut diagnostics, + ); let mut vars = match &task_use { TaskUse::Agent(name) => workflow .spec @@ -204,6 +217,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result ")), )] })?; + validate_parallel_memory_writes(workflow, &order, &tasks, file, &mut diagnostics); + if !diagnostics.is_empty() { + return Err(diagnostics); + } for task in tasks.values_mut() { task.predictability = match &task.uses { @@ -297,6 +315,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result usize { + 1 +} + +#[allow(clippy::too_many_arguments)] +fn task_memory_writes( + workflow: &Workflow, + task_use: &TaskUse, + input: &JsonMap, + declared: &[String], + file: &str, + position: usize, + diagnostics: &mut Vec, +) -> Vec { + let path = format!("spec.tasks[{position}].memoryWrites"); + let mut writes = BTreeSet::new(); + for key in declared { + if key.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "working-memory write keys must not be empty", + ) + .with_path(path.clone()), + ); + } else if !writes.insert(key.clone()) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("duplicate working-memory write key `{key}`"), + ) + .with_path(path.clone()), + ); + } + } + + let is_memory_write = match task_use { + TaskUse::Action(name) => workflow + .spec + .actions + .get(name) + .is_some_and(|action| action.kind == ActionKind::MemoryWrite), + TaskUse::Agent(_) => false, + }; + if !is_memory_write { + if !declared.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "memoryWrites is only valid for builtin.memory.write tasks", + ) + .with_path(path), + ); + } + return writes.into_iter().collect(); + } + + let Some(key) = input.get("key").and_then(Value::as_str) else { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "builtin.memory.write requires a string `key`", + ) + .with_path(format!("spec.tasks[{position}].with.key")), + ); + return writes.into_iter().collect(); + }; + if key.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "builtin.memory.write key must not be empty", + ) + .with_path(format!("spec.tasks[{position}].with.key")), + ); + } else if key.contains("${{") { + if writes.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "a templated builtin.memory.write key requires an explicit memoryWrites set", + ) + .with_path(path), + ); + } + } else if writes.is_empty() { + writes.insert(key.to_owned()); + } else if !writes.contains(key) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("literal working-memory key `{key}` is missing from memoryWrites"), + ) + .with_path(path), + ); + } + writes.into_iter().collect() +} + +fn validate_parallel_memory_writes( + workflow: &Workflow, + order: &[String], + tasks: &BTreeMap, + file: &str, + diagnostics: &mut Vec, +) { + if workflow.spec.runtime.max_concurrency == 1 { + return; + } + for (position, left_id) in order.iter().enumerate() { + let left = &tasks[left_id]; + for right_id in order.iter().skip(position + 1) { + let right = &tasks[right_id]; + if depends_on(left_id, right_id, tasks) || depends_on(right_id, left_id, tasks) { + continue; + } + let conflicts = left + .memory_writes + .iter() + .filter(|key| right.memory_writes.contains(key)) + .cloned() + .collect::>(); + if !conflicts.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "parallel tasks `{left_id}` and `{right_id}` have conflicting working-memory writes: {}", + conflicts.join(", ") + ), + ) + .with_path("spec.runtime.maxConcurrency") + .with_help( + "order the tasks with needs or give them disjoint memoryWrites keys", + ), + ); + } + } + } +} + +fn depends_on(task_id: &str, dependency_id: &str, tasks: &BTreeMap) -> bool { + let mut pending = tasks + .get(task_id) + .map_or_else(Vec::new, |task| task.needs.clone()); + let mut visited = BTreeSet::new(); + while let Some(candidate) = pending.pop() { + if candidate == dependency_id { + return true; + } + if visited.insert(candidate.clone()) + && let Some(task) = tasks.get(&candidate) + { + pending.extend(task.needs.iter().cloned()); + } + } + false +} + fn plan_requirements( workflow: &Workflow, tasks: &BTreeMap, @@ -1033,6 +1220,113 @@ spec: assert_eq!(plan.order, ["b", "a", "c"]); } + #[test] + fn parallel_memory_writes_are_inferred_and_conflicts_are_rejected() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-conflict } +spec: + runtime: { maxConcurrency: 2 } + actions: + remember: { kind: builtin.memory.write } + tasks: + - { id: left, uses: "action:remember", with: { key: shared, value: left } } + - { id: right, uses: "action:remember", with: { key: shared, value: right } } +"#, + ); + let diagnostics = compile(&workflow, "fixture.yaml").expect_err("conflict rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("parallel tasks `left` and `right`") + && diagnostic.message.contains("shared") + })); + } + + #[test] + fn ordered_or_disjoint_parallel_memory_writes_compile() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-memory } +spec: + runtime: { maxConcurrency: 3 } + actions: + remember: { kind: builtin.memory.write } + tasks: + - { id: left, uses: "action:remember", with: { key: left, value: one } } + - { id: right, uses: "action:remember", with: { key: right, value: two } } + - { id: ordered, uses: "action:remember", needs: [left], with: { key: left, value: three } } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("compiles"); + assert_eq!(plan.tasks["left"].memory_writes, ["left"]); + assert_eq!(plan.tasks["right"].memory_writes, ["right"]); + assert_eq!(plan.tasks["ordered"].memory_writes, ["left"]); + } + + #[test] + fn templated_memory_key_requires_declared_write_set() { + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: dynamic-memory-key } +spec: + inputs: + selected: { type: string } + actions: + remember: { kind: builtin.memory.write } + tasks: + - id: remember + uses: action:remember + with: { key: "${{ inputs.selected }}", value: kept } +"#; + let workflow = parse(source); + let diagnostics = compile(&workflow, "fixture.yaml").expect_err("declaration required"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("requires an explicit memoryWrites set") + })); + + let declared = source.replace( + "uses: action:remember", + "uses: action:remember\n memoryWrites: [selected]", + ); + compile(&parse(&declared), "fixture.yaml").expect("declared write compiles"); + } + + #[test] + fn additive_parallel_plan_fields_preserve_old_sequential_plans() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: old-plan } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - { id: one, uses: "action:assign" } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("compile"); + let mut json = serde_json::to_value(plan).expect("plan json"); + json.as_object_mut() + .expect("plan object") + .remove("maxConcurrency"); + json["tasks"]["one"] + .as_object_mut() + .expect("task object") + .remove("memoryWrites"); + let decoded: CompiledPlan = serde_json::from_value(json).expect("old plan decodes"); + assert_eq!(decoded.max_concurrency, 1); + assert!(decoded.tasks["one"].memory_writes.is_empty()); + } + #[test] fn rejects_cycle() { let workflow = parse( diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index dd5da32..f438101 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -250,6 +250,7 @@ pub const DEFAULT_PROCESS_STREAM_LIMIT_BYTES: u64 = 1024 * 1024; pub const DEFAULT_PROCESS_COMBINED_LIMIT_BYTES: u64 = 2 * 1024 * 1024; pub const MAX_PROCESS_OUTPUT_LIMIT_BYTES: u64 = 16 * 1024 * 1024; pub const MAX_PROCESS_TIMEOUT_SECONDS: u64 = 24 * 60 * 60; +pub const MAX_TASK_CONCURRENCY: usize = 64; impl ActionDefinition { #[must_use] @@ -419,6 +420,8 @@ pub struct TaskDefinition { pub uses: String, #[serde(default)] pub needs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub memory_writes: Vec, #[serde(default)] pub when: Option, #[serde(default)] @@ -889,17 +892,16 @@ fn validate_document(workflow: &Workflow, file: &str) -> Vec { .with_path("spec.tasks"), ); } - if workflow.spec.runtime.max_concurrency != 1 { + if workflow.spec.runtime.max_concurrency == 0 + || workflow.spec.runtime.max_concurrency > MAX_TASK_CONCURRENCY + { diagnostics.push( Diagnostic::error( DiagnosticCode::SchemaViolation, file, - "v1alpha1 requires runtime.maxConcurrency: 1", + format!("runtime.maxConcurrency must be between 1 and {MAX_TASK_CONCURRENCY}"), ) - .with_path("spec.runtime.maxConcurrency") - .with_help( - "parallel scheduling is deferred until deterministic merge semantics are versioned", - ), + .with_path("spec.runtime.maxConcurrency"), ); } if workflow.spec.runtime.default_timeout_seconds == 0 diff --git a/crates/agentctl-runtime/Cargo.toml b/crates/agentctl-runtime/Cargo.toml index f94fe0c..b07a738 100644 --- a/crates/agentctl-runtime/Cargo.toml +++ b/crates/agentctl-runtime/Cargo.toml @@ -15,6 +15,7 @@ agentctl-observability = { version = "0.2.0", path = "../agentctl-observability" agentctl-store = { version = "0.2.0", path = "../agentctl-store" } async-trait.workspace = true chrono.workspace = true +futures-util.workspace = true hex.workspace = true jsonschema.workspace = true serde.workspace = true diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index 87567ac..d24790f 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -25,11 +25,12 @@ use agentctl_observability::{NoopTraceSink, SpanKind, TraceEvent, TracePhase, Tr use agentctl_store::{ ApprovalRequest, ArtifactRecord, CheckpointRecord, EffectReconciliationRecord, EffectReconciliationRequest, LegacyTaskUpgrade, ReconciliationStatus, - ReusedTaskMaterialization, RunMode, SqliteStore, StoreError, TaskCompletionMetadata, - TaskDisposition, TaskExecutionMetadata, TaskRecord, + ReusedTaskMaterialization, RunMode, SqliteStore, StoreError, TaskBatchOutcome, TaskBatchResult, + TaskCompletionMetadata, TaskDisposition, TaskExecutionMetadata, TaskRecord, }; use async_trait::async_trait; use chrono::{DateTime, Utc}; +use futures_util::future::join_all; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; @@ -2334,6 +2335,24 @@ impl Runtime { trace_id: &str, options: RunOptions, cancellation: &CancellationToken, + ) -> Result { + let run = self.store.load_run(run_id)?; + let workflow: Workflow = serde_json::from_value(run.workflow)?; + if workflow.spec.runtime.max_concurrency == 1 { + self.drive_sequential(run_id, trace_id, options, cancellation) + .await + } else { + self.drive_parallel(run_id, trace_id, options, cancellation) + .await + } + } + + async fn drive_parallel( + &self, + run_id: &str, + trace_id: &str, + options: RunOptions, + cancellation: &CancellationToken, ) -> Result { loop { let run = self.store.load_run(run_id)?; @@ -2353,6 +2372,8 @@ impl Runtime { let failed = tasks.iter().any(|task| task.state == TaskState::Failed); let state = if failed { RunState::Failed + } else if tasks.iter().any(|task| task.state == TaskState::Cancelled) { + RunState::Cancelled } else { RunState::Succeeded }; @@ -2386,47 +2407,85 @@ impl Runtime { output: Some(output), }); } - let Some(task) = next_task(&run.plan, &tasks) else { - return Err(RuntimeError::InvalidState( - "no runnable task exists and the run is not terminal".to_owned(), - )); - }; - let dependencies: Vec<&TaskRecord> = task - .needs - .iter() - .filter_map(|needed| tasks.iter().find(|candidate| &candidate.task_id == needed)) - .collect(); - if dependencies.iter().any(|dependency| { - matches!( - dependency.state, - TaskState::Failed | TaskState::Cancelled | TaskState::Skipped - ) + + if let Some(failed) = tasks.iter().find(|record| { + record.state == TaskState::Failed + && run + .plan + .tasks + .get(&record.task_id) + .is_some_and(|task| task.failure == FailureBehavior::Stop) }) { - self.store.transition_task( - run_id, - &task.id, - TaskState::Skipped, - None, - Some("dependency did not succeed"), - None, - self.clock.now(), - trace_id, - )?; - continue; + if run.state == RunState::Running { + self.store.update_run_state( + run_id, + RunState::Failed, + None, + self.clock.now(), + trace_id, + )?; + } + return Err(RuntimeError::RunFailed { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + task: failed.task_id.clone(), + message: failed + .error + .clone() + .unwrap_or_else(|| "task failed".to_owned()), + }); } - let ready_state = tasks - .iter() - .find(|record| record.task_id == task.id) - .map(|record| record.state) - .ok_or_else(|| RuntimeError::InvalidState(format!("task `{}` missing", task.id)))?; - if ready_state == TaskState::Pending { - let context = context_for(&run, &tasks)?; + + let context = context_for(&run, &tasks)?; + let mut prepared_pending = false; + for task_id in &run.plan.order { + let Some(record) = tasks.iter().find(|record| &record.task_id == task_id) else { + return Err(RuntimeError::InvalidState(format!( + "task `{task_id}` missing" + ))); + }; + if record.state != TaskState::Pending { + continue; + } + let task = &run.plan.tasks[task_id]; + let dependencies = task + .needs + .iter() + .filter_map(|needed| { + tasks.iter().find(|candidate| &candidate.task_id == needed) + }) + .collect::>(); + if !dependencies + .iter() + .all(|dependency| dependency.state.is_terminal()) + { + continue; + } + if dependencies.iter().any(|dependency| { + matches!( + dependency.state, + TaskState::Failed | TaskState::Cancelled | TaskState::Skipped + ) + }) { + self.store.transition_task( + run_id, + task_id, + TaskState::Skipped, + None, + Some("dependency did not succeed"), + None, + self.clock.now(), + trace_id, + )?; + prepared_pending = true; + continue; + } if let Some(condition) = &task.when && !evaluate_when(condition, &context)? { self.store.transition_task( run_id, - &task.id, + task_id, TaskState::Skipped, Some(&serde_json::json!({"reason": "when condition was false"})), None, @@ -2434,11 +2493,12 @@ impl Runtime { self.clock.now(), trace_id, )?; + prepared_pending = true; continue; } self.store.transition_task( run_id, - &task.id, + task_id, TaskState::Ready, None, None, @@ -2446,46 +2506,119 @@ impl Runtime { self.clock.now(), trace_id, )?; + prepared_pending = true; + } + if prepared_pending { continue; } - if ready_state == TaskState::Ready { - self.store.transition_task( - run_id, - &task.id, - TaskState::Running, - None, - None, - None, - self.clock.now(), - trace_id, - )?; - self.trace( - TraceEvent::new( - SpanKind::Task, - TracePhase::Started, - "task.execute", - trace_id, - run_id, - self.clock.now(), + + let batch = ready_task_batch(&run.plan, &tasks, workflow.spec.runtime.max_concurrency); + if batch.is_empty() { + let retrying = tasks + .iter() + .filter(|task| task.state == TaskState::RetryScheduled) + .collect::>(); + if !retrying.is_empty() { + let backoff_ms = retrying + .iter() + .filter_map(|record| run.plan.tasks.get(&record.task_id)) + .map(|task| task.retry.backoff_ms) + .max() + .unwrap_or_default(); + tokio::select! { + () = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {} + () = cancellation.cancelled() => { + self.cancel_non_terminal(run_id, trace_id)?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Cancelled, + output: None, + }); + }, + } + for record in retrying { + self.store.transition_task( + run_id, + &record.task_id, + TaskState::Ready, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + } + continue; + } + if tasks.iter().any(|task| { + matches!( + task.state, + TaskState::WaitingForApproval | TaskState::WaitingForEffect ) - .task(&task.id), - )?; - continue; + }) { + if run.state == RunState::Running { + self.store.update_run_state( + run_id, + RunState::Paused, + None, + self.clock.now(), + trace_id, + )?; + } + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Paused, + output: None, + }); + } + return Err(RuntimeError::InvalidState( + "no runnable task exists and the run is not terminal".to_owned(), + )); } - if ready_state != TaskState::Running { - return Err(RuntimeError::InvalidState(format!( - "scheduler selected task `{}` in state {ready_state:?}", - task.id - ))); + + for task in &batch { + let record = tasks + .iter() + .find(|record| record.task_id == task.id) + .ok_or_else(|| { + RuntimeError::InvalidState(format!("task `{}` missing", task.id)) + })?; + if record.state == TaskState::Ready { + self.store.transition_task( + run_id, + &task.id, + TaskState::Running, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Task, + TracePhase::Started, + "task.execute", + trace_id, + run_id, + self.clock.now(), + ) + .task(&task.id) + .attributes( + serde_json::json!({ + "scheduler": "stable_parallel_batch", + "maxConcurrency": workflow.spec.runtime.max_concurrency, + }), + &[], + ), + )?; + } } - let current = self - .store - .list_tasks(run_id)? - .into_iter() - .find(|record| record.task_id == task.id) - .ok_or_else(|| RuntimeError::InvalidState(format!("task `{}` missing", task.id)))?; - let task_outputs = tasks + let running = self.store.list_tasks(run_id)?; + let task_outputs = running .iter() .filter_map(|record| { record @@ -2494,182 +2627,200 @@ impl Runtime { .map(|output| (record.task_id.clone(), output)) }) .collect::>(); - let execution_contract = if options.check { - serde_json::json!({}) - } else { - task_output_schema(&workflow, task).unwrap_or_else(|| serde_json::json!({})) - }; - let execution_metadata = TaskExecutionMetadata { - metadata_version: TASK_METADATA_VERSION, - definition_fingerprint: task_definition_fingerprint( - &workflow, task, &policy, None, - )?, - input_digest: resolved_input_digest( - run.inputs.as_object().ok_or_else(|| { - RuntimeError::InvalidState("run inputs must be an object".to_owned()) - })?, - &run.working_memory, - &task_outputs, + let mut prepared = Vec::with_capacity(batch.len()); + for task in batch { + let record = running + .iter() + .find(|record| record.task_id == task.id) + .cloned() + .ok_or_else(|| { + RuntimeError::InvalidState(format!("task `{}` missing", task.id)) + })?; + let execution_memory = record + .execution_memory + .clone() + .unwrap_or_else(|| run.working_memory.clone()); + let execution_contract = if options.check { + serde_json::json!({}) + } else { + task_output_schema(&workflow, task).unwrap_or_else(|| serde_json::json!({})) + }; + let execution_metadata = TaskExecutionMetadata { + metadata_version: TASK_METADATA_VERSION, + definition_fingerprint: task_definition_fingerprint( + &workflow, task, &policy, None, + )?, + input_digest: resolved_input_digest( + run.inputs.as_object().ok_or_else(|| { + RuntimeError::InvalidState("run inputs must be an object".to_owned()) + })?, + &execution_memory, + &task_outputs, + task, + )?, + output_contract_fingerprint: versioned_json_digest(&execution_contract)?, + }; + self.store.record_task_execution_metadata( + run_id, + &task.id, + &execution_metadata, + &execution_memory, + self.clock.now(), + )?; + let mut execution_run = run.clone(); + execution_run.working_memory = execution_memory; + prepared.push(PreparedBatchTask { task, - )?, - output_contract_fingerprint: versioned_json_digest(&execution_contract)?, - }; - self.store.record_task_execution_metadata( - run_id, - &task.id, - &execution_metadata, - self.clock.now(), - )?; - let execution = self - .execute_task( + record, + run: execution_run, + execution_contract, + execution_metadata, + }); + } + + let executions = join_all(prepared.iter().map(|prepared| async { + self.execute_task( &workflow, - &run, - ¤t, - task, + &prepared.run, + &prepared.record, + prepared.task, &policy, trace_id, options, cancellation, ) - .await; - let execution = execution.and_then(|execution| { - if let TaskExecution::Complete { output, .. } = &execution { - validate_output_contract(&execution_contract, output).map_err(|message| { - RuntimeError::Task { - task: task.id.clone(), - message: format!("task output contract failed: {message}"), - } - })?; - } - Ok(execution) - }); - match execution { - Ok(TaskExecution::Complete { output, memory }) => { - let effects = self.store.list_effects(run_id)?; - let delta = state_delta(&run.working_memory, memory.as_ref())?; - let completion = TaskCompletionMetadata { - execution: TaskExecutionMetadata { - definition_fingerprint: task_definition_fingerprint( - &workflow, - task, - &policy, - Some(&effects), - )?, - ..execution_metadata - }, - output_digest: versioned_json_digest(&output)?, - state_delta_digest: versioned_json_digest(&delta)?, - artifact_manifest: collect_artifacts( - &self.store, - &policy, - &effects, - run_id, - &task.id, - self.clock.now(), - )?, - state_delta: delta, - }; - self.store.complete_task( - run_id, - &task.id, - &output, - memory.as_ref(), - &completion, - self.clock.now(), - trace_id, - )?; - self.trace( - TraceEvent::new( - SpanKind::Task, - TracePhase::Completed, - "task.execute", - trace_id, - run_id, - self.clock.now(), - ) - .task(&task.id), - )?; - } - Ok(TaskExecution::Paused) => { - self.store.update_run_state( - run_id, - RunState::Paused, - None, - self.clock.now(), - trace_id, - )?; - self.trace( - TraceEvent::new( - SpanKind::Approval, - TracePhase::Waiting, - "approval.waiting", - trace_id, - run_id, - self.clock.now(), - ) - .task(&task.id), - )?; - return Ok(RunOutcome { - run_id: run_id.to_owned(), - trace_id: trace_id.to_owned(), - state: RunState::Paused, - output: None, - }); - } - Err(error) => { - if matches!(error, RuntimeError::Cancelled) { - self.cancel_non_terminal(run_id, trace_id)?; - return Ok(RunOutcome { - run_id: run_id.to_owned(), - trace_id: trace_id.to_owned(), - state: RunState::Cancelled, - output: None, + .await + .and_then(|execution| { + if let TaskExecution::Complete { output, .. } = &execution { + validate_output_contract(&prepared.execution_contract, output).map_err( + |message| RuntimeError::Task { + task: prepared.task.id.clone(), + message: format!("task output contract failed: {message}"), + }, + )?; + } + Ok(execution) + }) + })) + .await; + + if cancellation.is_cancelled() + || executions + .iter() + .any(|result| matches!(result, Err(RuntimeError::Cancelled))) + { + self.cancel_non_terminal(run_id, trace_id)?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Cancelled, + output: None, + }); + } + + let mut committed_memory = run.working_memory.clone(); + let mut has_memory_update = false; + let mut results = Vec::new(); + let mut paused = false; + let mut stop_failure = None; + let mut retrying = Vec::new(); + let all_effects = self.store.list_effects(run_id)?; + for (prepared, execution) in prepared.iter().zip(executions) { + match execution { + Ok(TaskExecution::Complete { output, memory }) => { + let delta = state_delta(&prepared.run.working_memory, memory.as_ref())?; + validate_memory_delta(prepared.task, &delta)?; + if memory.is_some() { + apply_state_delta(&mut committed_memory, &delta)?; + has_memory_update = true; + } + let completion = TaskCompletionMetadata { + execution: TaskExecutionMetadata { + definition_fingerprint: task_definition_fingerprint( + &workflow, + prepared.task, + &policy, + Some(&all_effects), + )?, + ..prepared.execution_metadata.clone() + }, + output_digest: versioned_json_digest(&output)?, + state_delta_digest: versioned_json_digest(&delta)?, + artifact_manifest: collect_artifacts( + &self.store, + &policy, + &all_effects, + run_id, + &prepared.task.id, + self.clock.now(), + )?, + state_delta: delta, + }; + results.push(TaskBatchResult { + task_id: prepared.task.id.clone(), + outcome: TaskBatchOutcome::Succeeded { + output, + metadata: Box::new(completion), + }, }); } - if current.attempt < task.retry.max_attempts && retryable_error(&error) { - self.store.transition_task( - run_id, - &task.id, - TaskState::RetryScheduled, - None, - Some(&error.to_string()), - None, - self.clock.now(), - trace_id, - )?; + Ok(TaskExecution::Paused) => paused = true, + Err(error) => { + let message = error.to_string(); + if prepared.record.attempt < prepared.task.retry.max_attempts + && retryable_error(&error) + { + results.push(TaskBatchResult { + task_id: prepared.task.id.clone(), + outcome: TaskBatchOutcome::RetryScheduled { + error: message.clone(), + }, + }); + retrying + .push((prepared.task.id.clone(), prepared.task.retry.backoff_ms)); + } else { + results.push(TaskBatchResult { + task_id: prepared.task.id.clone(), + outcome: TaskBatchOutcome::Failed { + error: message.clone(), + }, + }); + if prepared.task.failure == FailureBehavior::Stop + && stop_failure.is_none() + { + stop_failure = Some((prepared.task.id.clone(), message)); + } + } + } + } + } + + if !results.is_empty() { + self.store.commit_task_batch( + run_id, + &results, + has_memory_update.then_some(&committed_memory), + stop_failure.is_some(), + self.clock.now(), + trace_id, + )?; + } + for result in &results { + match &result.outcome { + TaskBatchOutcome::Succeeded { .. } => { self.trace( TraceEvent::new( - SpanKind::Retry, - TracePhase::Waiting, - "task.retry", + SpanKind::Task, + TracePhase::Completed, + "task.execute", trace_id, run_id, self.clock.now(), ) - .task(&task.id), - )?; - tokio::select! { - () = tokio::time::sleep(Duration::from_millis(task.retry.backoff_ms)) => {} - () = cancellation.cancelled() => { - self.cancel_non_terminal(run_id, trace_id)?; - return Ok(RunOutcome { - run_id: run_id.to_owned(), - trace_id: trace_id.to_owned(), - state: RunState::Cancelled, - output: None, - }); - }, - } - self.store.transition_task( - run_id, - &task.id, - TaskState::Ready, - None, - None, - None, - self.clock.now(), - trace_id, + .task(&result.task_id), )?; + } + TaskBatchOutcome::Failed { error } => { self.trace( TraceEvent::new( SpanKind::Task, @@ -2679,74 +2830,517 @@ impl Runtime { run_id, self.clock.now(), ) - .task(&task.id) - .attributes(serde_json::json!({"error": error.to_string()}), &[]), - )?; - } else { - self.store.transition_task( - run_id, - &task.id, - TaskState::Failed, - None, - Some(&error.to_string()), - None, - self.clock.now(), - trace_id, + .task(&result.task_id) + .attributes(serde_json::json!({"error": error}), &[]), )?; - if task.failure == FailureBehavior::Stop { - self.store.update_run_state( + } + TaskBatchOutcome::RetryScheduled { error } => { + self.trace( + TraceEvent::new( + SpanKind::Retry, + TracePhase::Waiting, + "task.retry", + trace_id, run_id, - RunState::Failed, - None, self.clock.now(), - trace_id, - )?; - return Err(RuntimeError::RunFailed { - run_id: run_id.to_owned(), - trace_id: trace_id.to_owned(), - task: task.id.clone(), - message: error.to_string(), - }); - } + ) + .task(&result.task_id) + .attributes(serde_json::json!({"error": error}), &[]), + )?; } } } + + if let Some((task, message)) = stop_failure { + return Err(RuntimeError::RunFailed { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + task, + message, + }); + } + if paused { + self.store.update_run_state( + run_id, + RunState::Paused, + None, + self.clock.now(), + trace_id, + )?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Paused, + output: None, + }); + } + if !retrying.is_empty() { + let backoff_ms = retrying + .iter() + .map(|(_, backoff_ms)| *backoff_ms) + .max() + .unwrap_or_default(); + tokio::select! { + () = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {} + () = cancellation.cancelled() => { + self.cancel_non_terminal(run_id, trace_id)?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Cancelled, + output: None, + }); + }, + } + for (task_id, _) in retrying { + self.store.transition_task( + run_id, + &task_id, + TaskState::Ready, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + } + } } } - #[allow(clippy::too_many_arguments)] - async fn execute_task( + async fn drive_sequential( &self, - workflow: &Workflow, - run: &agentctl_store::RunRecord, - record: &TaskRecord, - task: &agentctl_core::CompiledTask, - policy: &PolicyEngine, + run_id: &str, trace_id: &str, options: RunOptions, cancellation: &CancellationToken, - ) -> Result { - let tasks = self.store.list_tasks(&run.run_id)?; - let mut context = context_for(run, &tasks)?; - context.vars = task - .vars - .iter() - .map(|(name, value)| render(value, &context).map(|value| (name.clone(), value))) - .collect::, _>>()?; - let raw_input = serde_json::to_value(&task.input)?; - let input = render(&raw_input, &context)?; - match &task.uses { - TaskUse::Action(name) => { - let action = workflow.spec.actions.get(name).ok_or_else(|| { - RuntimeError::InvalidState(format!("action `{name}` disappeared after compile")) - })?; - self.execute_action( - workflow, - run, - record, - action, - input, - policy, + ) -> Result { + loop { + let run = self.store.load_run(run_id)?; + if run.cancellation_requested || cancellation.is_cancelled() { + self.cancel_non_terminal(run_id, trace_id)?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Cancelled, + output: None, + }); + } + let workflow: Workflow = serde_json::from_value(run.workflow.clone())?; + let policy = PolicyEngine::new(workflow.spec.policy.clone(), &self.base_path)?; + let tasks = self.store.list_tasks(run_id)?; + if tasks.iter().all(|task| task.state.is_terminal()) { + let failed = tasks.iter().any(|task| task.state == TaskState::Failed); + let state = if failed { + RunState::Failed + } else { + RunState::Succeeded + }; + let output = collect_outputs(&run, &tasks, &workflow.spec.outputs)?; + self.store.update_run_state( + run_id, + state, + Some(&output), + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Run, + if failed { + TracePhase::Failed + } else { + TracePhase::Completed + }, + "run.execute", + trace_id, + run_id, + self.clock.now(), + ) + .attributes(serde_json::json!({"state": state}), &[]), + )?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state, + output: Some(output), + }); + } + let Some(task) = next_task(&run.plan, &tasks) else { + return Err(RuntimeError::InvalidState( + "no runnable task exists and the run is not terminal".to_owned(), + )); + }; + let dependencies: Vec<&TaskRecord> = task + .needs + .iter() + .filter_map(|needed| tasks.iter().find(|candidate| &candidate.task_id == needed)) + .collect(); + if dependencies.iter().any(|dependency| { + matches!( + dependency.state, + TaskState::Failed | TaskState::Cancelled | TaskState::Skipped + ) + }) { + self.store.transition_task( + run_id, + &task.id, + TaskState::Skipped, + None, + Some("dependency did not succeed"), + None, + self.clock.now(), + trace_id, + )?; + continue; + } + let ready_state = tasks + .iter() + .find(|record| record.task_id == task.id) + .map(|record| record.state) + .ok_or_else(|| RuntimeError::InvalidState(format!("task `{}` missing", task.id)))?; + if ready_state == TaskState::Pending { + let context = context_for(&run, &tasks)?; + if let Some(condition) = &task.when + && !evaluate_when(condition, &context)? + { + self.store.transition_task( + run_id, + &task.id, + TaskState::Skipped, + Some(&serde_json::json!({"reason": "when condition was false"})), + None, + None, + self.clock.now(), + trace_id, + )?; + continue; + } + self.store.transition_task( + run_id, + &task.id, + TaskState::Ready, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + continue; + } + if ready_state == TaskState::Ready { + self.store.transition_task( + run_id, + &task.id, + TaskState::Running, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Task, + TracePhase::Started, + "task.execute", + trace_id, + run_id, + self.clock.now(), + ) + .task(&task.id), + )?; + continue; + } + if ready_state != TaskState::Running { + return Err(RuntimeError::InvalidState(format!( + "scheduler selected task `{}` in state {ready_state:?}", + task.id + ))); + } + + let current = self + .store + .list_tasks(run_id)? + .into_iter() + .find(|record| record.task_id == task.id) + .ok_or_else(|| RuntimeError::InvalidState(format!("task `{}` missing", task.id)))?; + let task_outputs = tasks + .iter() + .filter_map(|record| { + record + .output + .clone() + .map(|output| (record.task_id.clone(), output)) + }) + .collect::>(); + let execution_contract = if options.check { + serde_json::json!({}) + } else { + task_output_schema(&workflow, task).unwrap_or_else(|| serde_json::json!({})) + }; + let execution_metadata = TaskExecutionMetadata { + metadata_version: TASK_METADATA_VERSION, + definition_fingerprint: task_definition_fingerprint( + &workflow, task, &policy, None, + )?, + input_digest: resolved_input_digest( + run.inputs.as_object().ok_or_else(|| { + RuntimeError::InvalidState("run inputs must be an object".to_owned()) + })?, + &run.working_memory, + &task_outputs, + task, + )?, + output_contract_fingerprint: versioned_json_digest(&execution_contract)?, + }; + self.store.record_task_execution_metadata( + run_id, + &task.id, + &execution_metadata, + &run.working_memory, + self.clock.now(), + )?; + let execution = self + .execute_task( + &workflow, + &run, + ¤t, + task, + &policy, + trace_id, + options, + cancellation, + ) + .await; + let execution = execution.and_then(|execution| { + if let TaskExecution::Complete { output, .. } = &execution { + validate_output_contract(&execution_contract, output).map_err(|message| { + RuntimeError::Task { + task: task.id.clone(), + message: format!("task output contract failed: {message}"), + } + })?; + } + Ok(execution) + }); + match execution { + Ok(TaskExecution::Complete { output, memory }) => { + let effects = self.store.list_effects(run_id)?; + let delta = state_delta(&run.working_memory, memory.as_ref())?; + let completion = TaskCompletionMetadata { + execution: TaskExecutionMetadata { + definition_fingerprint: task_definition_fingerprint( + &workflow, + task, + &policy, + Some(&effects), + )?, + ..execution_metadata + }, + output_digest: versioned_json_digest(&output)?, + state_delta_digest: versioned_json_digest(&delta)?, + artifact_manifest: collect_artifacts( + &self.store, + &policy, + &effects, + run_id, + &task.id, + self.clock.now(), + )?, + state_delta: delta, + }; + self.store.complete_task( + run_id, + &task.id, + &output, + memory.as_ref(), + &completion, + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Task, + TracePhase::Completed, + "task.execute", + trace_id, + run_id, + self.clock.now(), + ) + .task(&task.id), + )?; + } + Ok(TaskExecution::Paused) => { + self.store.update_run_state( + run_id, + RunState::Paused, + None, + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Approval, + TracePhase::Waiting, + "approval.waiting", + trace_id, + run_id, + self.clock.now(), + ) + .task(&task.id), + )?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Paused, + output: None, + }); + } + Err(error) => { + if matches!(error, RuntimeError::Cancelled) { + self.cancel_non_terminal(run_id, trace_id)?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Cancelled, + output: None, + }); + } + if current.attempt < task.retry.max_attempts && retryable_error(&error) { + self.store.transition_task( + run_id, + &task.id, + TaskState::RetryScheduled, + None, + Some(&error.to_string()), + None, + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Retry, + TracePhase::Waiting, + "task.retry", + trace_id, + run_id, + self.clock.now(), + ) + .task(&task.id), + )?; + tokio::select! { + () = tokio::time::sleep(Duration::from_millis(task.retry.backoff_ms)) => {} + () = cancellation.cancelled() => { + self.cancel_non_terminal(run_id, trace_id)?; + return Ok(RunOutcome { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + state: RunState::Cancelled, + output: None, + }); + }, + } + self.store.transition_task( + run_id, + &task.id, + TaskState::Ready, + None, + None, + None, + self.clock.now(), + trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Task, + TracePhase::Failed, + "task.execute", + trace_id, + run_id, + self.clock.now(), + ) + .task(&task.id) + .attributes(serde_json::json!({"error": error.to_string()}), &[]), + )?; + } else { + self.store.transition_task( + run_id, + &task.id, + TaskState::Failed, + None, + Some(&error.to_string()), + None, + self.clock.now(), + trace_id, + )?; + if task.failure == FailureBehavior::Stop { + self.store.update_run_state( + run_id, + RunState::Failed, + None, + self.clock.now(), + trace_id, + )?; + return Err(RuntimeError::RunFailed { + run_id: run_id.to_owned(), + trace_id: trace_id.to_owned(), + task: task.id.clone(), + message: error.to_string(), + }); + } + } + } + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn execute_task( + &self, + workflow: &Workflow, + run: &agentctl_store::RunRecord, + record: &TaskRecord, + task: &agentctl_core::CompiledTask, + policy: &PolicyEngine, + trace_id: &str, + options: RunOptions, + cancellation: &CancellationToken, + ) -> Result { + let tasks = self.store.list_tasks(&run.run_id)?; + let mut context = context_for(run, &tasks)?; + context.vars = task + .vars + .iter() + .map(|(name, value)| render(value, &context).map(|value| (name.clone(), value))) + .collect::, _>>()?; + let raw_input = serde_json::to_value(&task.input)?; + let input = render(&raw_input, &context)?; + match &task.uses { + TaskUse::Action(name) => { + let action = workflow.spec.actions.get(name).ok_or_else(|| { + RuntimeError::InvalidState(format!("action `{name}` disappeared after compile")) + })?; + if action.kind == ActionKind::MemoryWrite { + let key = required_string(&input, "key")?; + if (workflow.spec.runtime.max_concurrency > 1 || !task.memory_writes.is_empty()) + && !task.memory_writes.contains(&key) + { + return Err(RuntimeError::Task { + task: task.id.clone(), + message: format!( + "resolved working-memory key `{key}` is not declared in memoryWrites" + ), + }); + } + } + self.execute_action( + workflow, + run, + record, + action, + input, + policy, trace_id, options, cancellation, @@ -3933,35 +4527,6 @@ impl Runtime { Err(StoreError::EffectNotFound(_)) => {} Err(error) => return Err(RuntimeError::Store(error)), } - self.store - .record_effect_request(request, self.clock.now())?; - self.trace( - TraceEvent::new( - match request.effect_class { - EffectClass::Model => SpanKind::ProviderRequest, - EffectClass::RemoteAgent => SpanKind::A2aDelegation, - EffectClass::Network if request.operation.starts_with("mcp.") => { - SpanKind::McpRequest - } - _ => SpanKind::Effect, - }, - TracePhase::Started, - &request.operation, - &request.trace_id, - &request.run_id, - self.clock.now(), - ) - .task(&request.task_id) - .effect(&request.id) - .attributes( - serde_json::json!({ - "inputDigest": request.input_digest, - "effectClass": request.effect_class, - "risk": request.risk, - }), - &[], - ), - )?; let context = PolicyContext { run_id: request.run_id.clone(), trace_id: request.trace_id.clone(), @@ -3979,43 +4544,72 @@ impl Runtime { }; let decision = policy.decide_with_approval(&context, approval); match decision { - PolicyDecision::Allow { .. } => Ok(PreparedEffect::Execute), + PolicyDecision::Allow { .. } => { + self.store + .record_effect_request(request, self.clock.now())?; + self.trace_effect_request(request)?; + Ok(PreparedEffect::Execute) + } PolicyDecision::Deny { reason } => Err(RuntimeError::Task { task: request.task_id.clone(), message: format!("policy denied effect: {reason}"), }), PolicyDecision::RequireApproval { reason } => { let approval_id = format!("approval-{}", &request.id[..16]); - self.store.create_approval(&ApprovalRequest { - approval_id, - run_id: request.run_id.clone(), - effect_id: request.id.clone(), - task_id: request.task_id.clone(), - agent: agent.map(ToOwned::to_owned), - tool: tool.to_owned(), - capability: capability.to_owned(), - risk: format!("{:?}", request.risk).to_ascii_lowercase(), - redacted_input: redact(&request.input, &[]), - expected_effect: request.expected_effect.clone(), - reason, - trace_id: request.trace_id.clone(), - requested_at: self.clock.now(), - })?; - self.store.transition_task( - &request.run_id, - &request.task_id, - TaskState::WaitingForApproval, - None, - None, - None, - self.clock.now(), - &request.trace_id, + self.store.create_approval( + request, + &ApprovalRequest { + approval_id, + run_id: request.run_id.clone(), + effect_id: request.id.clone(), + task_id: request.task_id.clone(), + agent: agent.map(ToOwned::to_owned), + tool: tool.to_owned(), + capability: capability.to_owned(), + risk: format!("{:?}", request.risk).to_ascii_lowercase(), + redacted_input: redact(&request.input, &[]), + expected_effect: request.expected_effect.clone(), + reason, + trace_id: request.trace_id.clone(), + requested_at: self.clock.now(), + }, )?; + self.trace_effect_request(request)?; Ok(PreparedEffect::Paused) } } } + fn trace_effect_request(&self, request: &EffectRequest) -> Result<(), RuntimeError> { + self.trace( + TraceEvent::new( + match request.effect_class { + EffectClass::Model => SpanKind::ProviderRequest, + EffectClass::RemoteAgent => SpanKind::A2aDelegation, + EffectClass::Network if request.operation.starts_with("mcp.") => { + SpanKind::McpRequest + } + _ => SpanKind::Effect, + }, + TracePhase::Started, + &request.operation, + &request.trace_id, + &request.run_id, + self.clock.now(), + ) + .task(&request.task_id) + .effect(&request.id) + .attributes( + serde_json::json!({ + "inputDigest": request.input_digest, + "effectClass": request.effect_class, + "risk": request.risk, + }), + &[], + ), + ) + } + fn cancel_non_terminal(&self, run_id: &str, trace_id: &str) -> Result<(), RuntimeError> { for task in self.store.list_tasks(run_id)? { if !task.state.is_terminal() { @@ -4079,6 +4673,14 @@ enum TaskExecution { Paused, } +struct PreparedBatchTask<'a> { + task: &'a agentctl_core::CompiledTask, + record: TaskRecord, + run: agentctl_store::RunRecord, + execution_contract: Value, + execution_metadata: TaskExecutionMetadata, +} + fn repair_block( task_id: &str, rule: &str, @@ -4957,6 +5559,72 @@ fn next_task<'a>( }) } +fn ready_task_batch<'a>( + plan: &'a CompiledPlan, + records: &[TaskRecord], + limit: usize, +) -> Vec<&'a agentctl_core::CompiledTask> { + plan.order + .iter() + .filter_map(|id| { + let record = records.iter().find(|record| &record.task_id == id)?; + if !matches!(record.state, TaskState::Ready | TaskState::Running) { + return None; + } + let task = plan.tasks.get(id)?; + task.needs + .iter() + .all(|needed| { + records + .iter() + .find(|record| &record.task_id == needed) + .is_some_and(|record| record.state.is_terminal()) + }) + .then_some(task) + }) + .take(limit) + .collect() +} + +fn validate_memory_delta( + task: &agentctl_core::CompiledTask, + delta: &Value, +) -> Result<(), RuntimeError> { + let mut changed = delta + .get("set") + .and_then(Value::as_object) + .map(|set| set.keys().cloned().collect::>()) + .ok_or_else(|| RuntimeError::InvalidState("state delta has no set object".to_owned()))?; + let removed = delta + .get("remove") + .and_then(Value::as_array) + .ok_or_else(|| RuntimeError::InvalidState("state delta has no remove array".to_owned()))?; + for key in removed { + changed.insert( + key.as_str() + .ok_or_else(|| { + RuntimeError::InvalidState("state delta key is not a string".to_owned()) + })? + .to_owned(), + ); + } + let undeclared = changed + .difference(&task.memory_writes.iter().cloned().collect()) + .cloned() + .collect::>(); + if undeclared.is_empty() { + Ok(()) + } else { + Err(RuntimeError::Task { + task: task.id.clone(), + message: format!( + "task changed undeclared working-memory key(s): {}", + undeclared.join(", ") + ), + }) + } +} + fn context_for( run: &agentctl_store::RunRecord, tasks: &[TaskRecord], @@ -5181,8 +5849,9 @@ mod tests { use agentctl_core::tool::{ToolContract, ToolContractError, ToolExecutor}; use agentctl_observability::BufferedTraceSink; use agentctl_store::ApprovalResolution; - use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; + use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}; use tempfile::tempdir; + use tokio::sync::Notify; struct FixedClock; @@ -5210,53 +5879,214 @@ mod tests { fn now(&self) -> DateTime { DateTime::from_timestamp(self.0.load(Ordering::SeqCst), 0).unwrap_or_else(Utc::now) } - } - - #[derive(Default)] - struct SequenceIds(AtomicU64); + } + + #[derive(Default)] + struct SequenceIds(AtomicU64); + + impl IdGenerator for SequenceIds { + fn next_id(&self, kind: &str) -> String { + format!("{kind}-{}", self.0.fetch_add(1, Ordering::SeqCst) + 1) + } + } + + #[derive(Default)] + struct CountingProvider(AtomicU64); + + #[async_trait] + impl ModelProvider for CountingProvider { + fn name(&self) -> &'static str { + "fake" + } + + async fn complete( + &self, + _request: &ProviderRequest, + cancellation: &CancellationToken, + ) -> Result { + if cancellation.is_cancelled() { + return Err(ProviderError::Cancelled); + } + let call = self.0.fetch_add(1, Ordering::SeqCst) + 1; + Ok(ProviderResponse { + response_id: Some(format!("fake-{call}")), + text: format!("answer-{call}"), + tool_calls: Vec::new(), + assistant_content: vec![ContentBlock::Text { + text: format!("answer-{call}"), + }], + continuation: None, + usage: Usage { + input_tokens: 2, + output_tokens: 1, + ..Usage::default() + }, + finish_reason: FinishReason::Complete, + }) + } + } + + struct OverlapProvider { + active: AtomicUsize, + peak: AtomicUsize, + delay: Duration, + } + + impl OverlapProvider { + fn new(delay: Duration) -> Self { + Self { + active: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + delay, + } + } + } + + #[async_trait] + impl ModelProvider for OverlapProvider { + fn name(&self) -> &'static str { + "fake" + } + + async fn complete( + &self, + request: &ProviderRequest, + cancellation: &CancellationToken, + ) -> Result { + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(active, Ordering::SeqCst); + tokio::select! { + () = tokio::time::sleep(self.delay) => {} + () = cancellation.cancelled() => { + self.active.fetch_sub(1, Ordering::SeqCst); + return Err(ProviderError::Cancelled); + } + } + self.active.fetch_sub(1, Ordering::SeqCst); + let text = request_text(request); + Ok(ProviderResponse { + response_id: Some(format!("overlap-{text}")), + text: text.clone(), + tool_calls: Vec::new(), + assistant_content: vec![ContentBlock::Text { text }], + continuation: None, + usage: Usage::default(), + finish_reason: FinishReason::Complete, + }) + } + } + + #[derive(Default)] + struct CancellationProvider { + started: AtomicUsize, + notify: Notify, + } + + #[async_trait] + impl ModelProvider for CancellationProvider { + fn name(&self) -> &'static str { + "fake" + } + + async fn complete( + &self, + _request: &ProviderRequest, + cancellation: &CancellationToken, + ) -> Result { + self.started.fetch_add(1, Ordering::SeqCst); + self.notify.notify_waiters(); + cancellation.cancelled().await; + Err(ProviderError::Cancelled) + } + } + + struct FailureSiblingProvider; + + #[async_trait] + impl ModelProvider for FailureSiblingProvider { + fn name(&self) -> &'static str { + "fake" + } - impl IdGenerator for SequenceIds { - fn next_id(&self, kind: &str) -> String { - format!("{kind}-{}", self.0.fetch_add(1, Ordering::SeqCst) + 1) + async fn complete( + &self, + request: &ProviderRequest, + cancellation: &CancellationToken, + ) -> Result { + let text = request_text(request); + if text == "fail" { + return Err(ProviderError::Malformed( + "deterministic parallel failure".to_owned(), + )); + } + tokio::select! { + () = tokio::time::sleep(Duration::from_millis(40)) => {} + () = cancellation.cancelled() => return Err(ProviderError::Cancelled), + } + Ok(ProviderResponse { + response_id: Some("sibling-success".to_owned()), + text: text.clone(), + tool_calls: Vec::new(), + assistant_content: vec![ContentBlock::Text { text }], + continuation: None, + usage: Usage::default(), + finish_reason: FinishReason::Complete, + }) } } #[derive(Default)] - struct CountingProvider(AtomicU64); + struct ParallelRetryProvider { + broken_calls: AtomicUsize, + sibling_calls: AtomicUsize, + } #[async_trait] - impl ModelProvider for CountingProvider { + impl ModelProvider for ParallelRetryProvider { fn name(&self) -> &'static str { "fake" } async fn complete( &self, - _request: &ProviderRequest, - cancellation: &CancellationToken, + request: &ProviderRequest, + _cancellation: &CancellationToken, ) -> Result { - if cancellation.is_cancelled() { - return Err(ProviderError::Cancelled); + let prompt = request_text(request); + if prompt == "broken" && self.broken_calls.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(ProviderError::Malformed( + "first parallel attempt fails".to_owned(), + )); + } + if prompt == "sibling" { + let prior = self.sibling_calls.fetch_add(1, Ordering::SeqCst); + assert_eq!(prior, 0, "successful sibling must be reused by retry"); } - let call = self.0.fetch_add(1, Ordering::SeqCst) + 1; Ok(ProviderResponse { - response_id: Some(format!("fake-{call}")), - text: format!("answer-{call}"), + response_id: Some(format!("parallel-retry-{prompt}")), + text: prompt.clone(), tool_calls: Vec::new(), - assistant_content: vec![ContentBlock::Text { - text: format!("answer-{call}"), - }], + assistant_content: vec![ContentBlock::Text { text: prompt }], continuation: None, - usage: Usage { - input_tokens: 2, - output_tokens: 1, - ..Usage::default() - }, + usage: Usage::default(), finish_reason: FinishReason::Complete, }) } } + fn request_text(request: &ProviderRequest) -> String { + match request.messages.first() { + Some(Message::User(content)) => content + .first() + .and_then(|block| match block { + ContentBlock::Text { text } => Some(text.clone()), + _ => None, + }) + .unwrap_or_default(), + _ => String::new(), + } + } + struct PromptEchoProvider; #[async_trait] @@ -5615,64 +6445,577 @@ mod tests { } } - #[async_trait] - impl ToolExecutor for SingleUseRepairTool { - fn contract(&self) -> &ToolContract { - self.inner.contract() - } - - async fn execute( - &self, - input: Value, - cancellation: &CancellationToken, - ) -> Result { - assert_eq!( - self.calls.fetch_add(1, Ordering::SeqCst), - 0, - "reused upstream tool must not execute during repair" - ); - self.inner.execute(input, cancellation).await - } + #[async_trait] + impl ToolExecutor for SingleUseRepairTool { + fn contract(&self) -> &ToolContract { + self.inner.contract() + } + + async fn execute( + &self, + input: Value, + cancellation: &CancellationToken, + ) -> Result { + assert_eq!( + self.calls.fetch_add(1, Ordering::SeqCst), + 0, + "reused upstream tool must not execute during repair" + ); + self.inner.execute(input, cancellation).await + } + } + + struct PanicTool { + contract: ToolContract, + } + + #[async_trait] + impl ToolExecutor for PanicTool { + fn contract(&self) -> &ToolContract { + &self.contract + } + + async fn execute( + &self, + _input: Value, + _cancellation: &CancellationToken, + ) -> Result { + panic!("tool executor must not run during recorded replay") + } + } + + struct RejectReconciliationHook; + + impl EffectReconciliationHook for RejectReconciliationHook { + fn validate( + &self, + _effect: &EffectRecord, + _evidence: &Value, + _result: Option<&Value>, + ) -> Result<(), String> { + Err("external verifier did not confirm the record".to_owned()) + } + } + + fn compile_fixture(source: &str) -> (Workflow, CompiledPlan) { + let workflow = parse_workflow(source, "fixture.yaml") + .expect("parse fixture") + .workflow; + let plan = compile(&workflow, "fixture.yaml").expect("compile fixture"); + (workflow, plan) + } + + #[tokio::test] + async fn parallel_scheduler_overlaps_work_caps_concurrency_and_commits_in_plan_order() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(OverlapProvider::new(Duration::from_millis(40))); + let runtime = runtime(store.clone(), directory.path()) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-overlap } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: return the prompt + maxTurns: 1 + tasks: + - { id: first, uses: "agent:worker", with: { prompt: first } } + - { id: second, uses: "agent:worker", with: { prompt: second } } + - { id: third, uses: "agent:worker", with: { prompt: third } } +"#, + ); + let outcome = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("parallel run"); + assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!(provider.peak.load(Ordering::SeqCst), 2); + let tasks = store.list_tasks(&outcome.run_id).expect("tasks"); + assert_eq!(tasks[0].output.as_ref().expect("first")["text"], "first"); + assert_eq!(tasks[1].output.as_ref().expect("second")["text"], "second"); + assert_eq!(tasks[2].output.as_ref().expect("third")["text"], "third"); + let completion_order = store + .audit_events(&outcome.run_id) + .expect("audit") + .into_iter() + .filter(|event| { + event.event_type == "task.transition" && event.payload["to"] == "succeeded" + }) + .filter_map(|event| event.task_id) + .collect::>(); + assert_eq!(completion_order, ["first", "second", "third"]); + } + + #[tokio::test] + async fn parallel_memory_deltas_merge_atomically_and_replay_offline() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-memory } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + memory: + working: { seed: kept } + actions: + remember: { kind: builtin.memory.write } + tasks: + - { id: left, uses: "action:remember", with: { key: left, value: one } } + - { id: right, uses: "action:remember", with: { key: right, value: two } } +"#, + ); + let runtime = runtime(store.clone(), directory.path()); + let outcome = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("parallel memory run"); + let source = store.load_run(&outcome.run_id).expect("source run"); + assert_eq!( + source.working_memory, + serde_json::json!({"seed": "kept", "left": "one", "right": "two"}) + ); + let replay = runtime + .replay(&outcome.run_id) + .await + .expect("offline replay"); + let replayed = store.load_run(&replay.run_id).expect("replayed run"); + assert_eq!(replayed.working_memory, source.working_memory); + assert!( + store + .list_effects(&replay.run_id) + .expect("effects") + .is_empty() + ); + } + + #[tokio::test] + async fn parallel_dynamic_memory_write_is_rejected_before_effect_dispatch() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-write-declaration } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + inputs: + selected: { type: string } + actions: + remember: { kind: builtin.memory.write } + tasks: + - id: remember + uses: action:remember + memoryWrites: [allowed] + with: { key: "${{ inputs.selected }}", value: blocked } +"#, + ); + let run_id = match runtime(store.clone(), directory.path()) + .start( + &workflow, + &plan, + serde_json::json!({"selected": "undeclared"}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { + run_id, message, .. + }) => { + assert!(message.contains("not declared in memoryWrites")); + run_id + } + other => panic!("expected declared-write failure, got {other:?}"), + }; + assert!(store.list_effects(&run_id).expect("effects").is_empty()); + } + + #[tokio::test] + async fn parallel_cancellation_propagates_to_every_running_task() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(CancellationProvider::default()); + let runtime = runtime(store.clone(), directory.path()) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-cancellation } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: wait + maxTurns: 1 + tasks: + - { id: first, uses: "agent:worker", with: { prompt: first } } + - { id: second, uses: "agent:worker", with: { prompt: second } } +"#, + ); + let cancellation = CancellationToken::new(); + let run_cancellation = cancellation.clone(); + let handle = tokio::spawn(async move { + runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &run_cancellation, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), async { + while provider.started.load(Ordering::SeqCst) != 2 { + provider.notify.notified().await; + } + }) + .await + .expect("both tasks started"); + cancellation.cancel(); + let outcome = handle.await.expect("join").expect("cancel outcome"); + assert_eq!(outcome.state, RunState::Cancelled); + assert!( + store + .list_tasks(&outcome.run_id) + .expect("tasks") + .iter() + .all(|task| task.state == TaskState::Cancelled) + ); + } + + #[tokio::test] + async fn stop_failure_waits_for_and_commits_successful_parallel_sibling() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()).with_registry( + RuntimeRegistry::default().with_provider("fake", Arc::new(FailureSiblingProvider)), + ); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-failure } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: execute + maxTurns: 1 + tasks: + - { id: broken, uses: "agent:worker", with: { prompt: fail } } + - { id: sibling, uses: "agent:worker", with: { prompt: keep } } +"#, + ); + let run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, task, .. }) => { + assert_eq!(task, "broken"); + run_id + } + other => panic!("expected stop failure, got {other:?}"), + }; + let tasks = store.list_tasks(&run_id).expect("tasks"); + assert_eq!(tasks[0].state, TaskState::Failed); + assert_eq!(tasks[1].state, TaskState::Succeeded); + assert_eq!( + tasks[1].output.as_ref().expect("sibling output")["text"], + "keep" + ); + assert_eq!( + store.load_run(&run_id).expect("run").state, + RunState::Failed + ); } - struct PanicTool { - contract: ToolContract, + #[tokio::test] + async fn continue_failure_preserves_independent_parallel_branch() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()).with_registry( + RuntimeRegistry::default().with_provider("fake", Arc::new(FailureSiblingProvider)), + ); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-continue } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: execute + maxTurns: 1 + tasks: + - id: broken + uses: agent:worker + failure: continue + with: { prompt: fail } + - { id: sibling, uses: "agent:worker", with: { prompt: keep } } +"#, + ); + let outcome = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("continue returns terminal outcome"); + assert_eq!(outcome.state, RunState::Failed); + let tasks = store.list_tasks(&outcome.run_id).expect("tasks"); + assert_eq!(tasks[0].state, TaskState::Failed); + assert_eq!(tasks[1].state, TaskState::Succeeded); } - #[async_trait] - impl ToolExecutor for PanicTool { - fn contract(&self) -> &ToolContract { - &self.contract - } - - async fn execute( - &self, - _input: Value, - _cancellation: &CancellationToken, - ) -> Result { - panic!("tool executor must not run during recorded replay") + #[tokio::test] + async fn parallel_approvals_pause_and_resume_all_tasks() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-approvals } +spec: + runtime: { maxConcurrency: 2 } + policy: + approval: always + nonInteractive: pause + actions: + remember: { kind: builtin.memory.write } + tasks: + - { id: left, uses: "action:remember", with: { key: left, value: one } } + - { id: right, uses: "action:remember", with: { key: right, value: two } } +"#, + ); + let runtime = runtime(store.clone(), directory.path()); + let paused = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("paused run"); + assert_eq!(paused.state, RunState::Paused); + let tasks = store.list_tasks(&paused.run_id).expect("paused tasks"); + assert!(tasks.iter().all(|task| { + task.state == TaskState::WaitingForApproval && task.execution_memory.is_some() + })); + let approvals = store.pending_approvals(&paused.run_id).expect("approvals"); + assert_eq!(approvals.len(), 2); + for approval in approvals { + store + .resolve_approval( + &approval.approval_id, + ApprovalResolution::Approved, + "test", + "approved parallel task", + FixedClock.now(), + ) + .expect("approval"); } + let resumed = runtime + .resume( + &paused.run_id, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("resume"); + assert_eq!(resumed.state, RunState::Succeeded); + assert_eq!( + store.load_run(&resumed.run_id).expect("run").working_memory, + serde_json::json!({"left": "one", "right": "two"}) + ); } - struct RejectReconciliationHook; - - impl EffectReconciliationHook for RejectReconciliationHook { - fn validate( - &self, - _effect: &EffectRecord, - _evidence: &Value, - _result: Option<&Value>, - ) -> Result<(), String> { - Err("external verifier did not confirm the record".to_owned()) - } + #[tokio::test] + async fn failed_only_retry_reuses_successful_parallel_sibling() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(ParallelRetryProvider::default()); + let runtime = runtime(store.clone(), directory.path()) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-retry } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: execute + maxTurns: 1 + tasks: + - { id: broken, uses: "agent:worker", with: { prompt: broken } } + - { id: sibling, uses: "agent:worker", with: { prompt: sibling } } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let retry_plan = runtime + .plan_retry(&source_run_id, &workflow, &plan, &[], true, false) + .expect("retry plan"); + assert!(retry_plan.compatible, "{:?}", retry_plan.blocked_reuse); + assert_eq!(retry_plan.retry_roots, ["broken"]); + assert_eq!(retry_plan.reused_tasks, ["sibling"]); + let outcome = runtime + .retry( + &workflow, + &plan, + retry_plan, + Some("retry failed parallel branch"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("retry"); + assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!(provider.broken_calls.load(Ordering::SeqCst), 2); + assert_eq!(provider.sibling_calls.load(Ordering::SeqCst), 1); + let retry_tasks = store.list_tasks(&outcome.run_id).expect("retry tasks"); + assert_eq!(retry_tasks[0].disposition, TaskDisposition::Executed); + assert_eq!(retry_tasks[1].disposition, TaskDisposition::Reused); } - fn compile_fixture(source: &str) -> (Workflow, CompiledPlan) { - let workflow = parse_workflow(source, "fixture.yaml") - .expect("parse fixture") - .workflow; - let plan = compile(&workflow, "fixture.yaml").expect("compile fixture"); - (workflow, plan) + #[tokio::test] + async fn repair_reuses_successful_parallel_memory_branch() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: parallel-repair } +spec: + runtime: { maxConcurrency: 2 } + policy: { approval: never } + actions: + remember: { kind: builtin.memory.write } + verify: { kind: builtin.assert } + tasks: + - { id: remember, uses: "action:remember", with: { key: durable, value: kept } } + - { id: verify, uses: "action:verify", with: { that: false } } +"#; + let target = source.replace("that: false", "that: true"); + let (source_workflow, source_plan) = compile_fixture(source); + let source_run_id = match runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected source failure, got {other:?}"), + }; + let (target_workflow, target_plan) = compile_fixture(&target); + let repair_plan = runtime + .plan_repair( + &source_run_id, + &target_workflow, + &target_plan, + &["verify".to_owned()], + false, + ) + .expect("repair plan"); + assert!(repair_plan.compatible, "{:?}", repair_plan.blocked_reuse); + assert_eq!(repair_plan.reused_tasks, ["remember"]); + assert_eq!(repair_plan.rerun_tasks, ["verify"]); + let outcome = runtime + .repair( + &target_workflow, + &target_plan, + repair_plan, + Some("fix independent assertion"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair"); + assert_eq!(outcome.state, RunState::Succeeded); + assert_eq!( + store + .load_run(&outcome.run_id) + .expect("repair run") + .working_memory, + serde_json::json!({"durable": "kept"}) + ); + let tasks = store.list_tasks(&outcome.run_id).expect("repair tasks"); + assert_eq!(tasks[0].disposition, TaskDisposition::Reused); + assert_eq!(tasks[1].disposition, TaskDisposition::Executed); } #[test] diff --git a/crates/agentctl-store/src/encryption.rs b/crates/agentctl-store/src/encryption.rs index 7212754..418bbc7 100644 --- a/crates/agentctl-store/src/encryption.rs +++ b/crates/agentctl-store/src/encryption.rs @@ -74,6 +74,10 @@ pub(crate) const SENSITIVE_COLUMNS: &[SensitiveColumn] = &[ table: "task_states", column: "reuse_decision_json", }, + SensitiveColumn { + table: "task_states", + column: "execution_memory_json", + }, SensitiveColumn { table: "effects", column: "input_json", diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index 97ef2c3..a15d497 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -25,7 +25,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use thiserror::Error; -pub const DATABASE_SCHEMA_VERSION: u32 = 10; +pub const DATABASE_SCHEMA_VERSION: u32 = 11; pub const RUNTIME_STATE_VERSION: u32 = 1; pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; pub const AUDIT_EVENT_VERSION: u32 = 1; @@ -318,6 +318,10 @@ CREATE TABLE state_encryption ( ); "#; +const MIGRATION_11: &str = r#" +ALTER TABLE task_states ADD COLUMN execution_memory_json TEXT; +"#; + #[derive(Clone)] pub struct SqliteStore { connection: Arc>, @@ -479,6 +483,8 @@ pub struct TaskRecord { pub state_delta_digest: Option, pub artifact_manifest: Vec, pub reuse_decision: Option, + #[serde(skip)] + pub execution_memory: Option, pub updated_at: DateTime, } @@ -501,6 +507,26 @@ pub struct TaskCompletionMetadata { pub artifact_manifest: Vec, } +#[derive(Debug, Clone, PartialEq)] +pub enum TaskBatchOutcome { + Succeeded { + output: Value, + metadata: Box, + }, + Failed { + error: String, + }, + RetryScheduled { + error: String, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TaskBatchResult { + pub task_id: String, + pub outcome: TaskBatchOutcome, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ReusedTaskMaterialization { @@ -1546,7 +1572,7 @@ impl SqliteStore { pub fn list_tasks(&self, run_id: &str) -> Result, StoreError> { let connection = self.connection.lock(); let mut statement = connection.prepare( - "SELECT task_id, position, state, attempt, output_json, error, updated_at, disposition, metadata_version, source_run_id, source_task_id, source_attempt, definition_fingerprint, input_digest, output_contract_fingerprint, output_digest, state_delta_json, state_delta_digest, artifact_manifest_json, reuse_decision_json FROM task_states WHERE run_id = ?1 ORDER BY position", + "SELECT task_id, position, state, attempt, output_json, error, updated_at, disposition, metadata_version, source_run_id, source_task_id, source_attempt, definition_fingerprint, input_digest, output_contract_fingerprint, output_digest, state_delta_json, state_delta_digest, artifact_manifest_json, reuse_decision_json, execution_memory_json FROM task_states WHERE run_id = ?1 ORDER BY position", )?; let rows = statement.query_map([run_id], |row| { Ok(( @@ -1570,6 +1596,7 @@ impl SqliteStore { row.get::<_, Option>(17)?, row.get::<_, Option>(18)?, row.get::<_, Option>(19)?, + row.get::<_, Option>(20)?, )) })?; rows.map(|row| { @@ -1621,6 +1648,16 @@ impl SqliteStore { ) }) .transpose()?, + execution_memory: row + .20 + .map(|value| { + decode_protected( + &self.protection, + &value, + "task_states.execution_memory_json", + ) + }) + .transpose()?, updated_at: parse_time(&row.6, "task.updated_at")?, }) }) @@ -1657,7 +1694,7 @@ impl SqliteStore { .transition(next) .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; transaction.execute( - "UPDATE task_states SET state = ?3, output_json = COALESCE(?4, output_json), error = ?5, attempt = attempt + ?7, updated_at = ?6 WHERE run_id = ?1 AND task_id = ?2", + "UPDATE task_states SET state = ?3, output_json = COALESCE(?4, output_json), error = ?5, attempt = attempt + ?7, execution_memory_json = CASE WHEN ?8 = 1 THEN NULL ELSE execution_memory_json END, updated_at = ?6 WHERE run_id = ?1 AND task_id = ?2", params![ run_id, task_id, @@ -1673,7 +1710,8 @@ impl SqliteStore { .map(|value| protect_text(&self.protection, value, "task_states.error")) .transpose()?, now.to_rfc3339(), - i64::from(current == TaskState::Ready && next == TaskState::Running) + i64::from(current == TaskState::Ready && next == TaskState::Running), + i64::from(current == TaskState::RetryScheduled && next == TaskState::Ready), ], )?; if let Some(memory) = working_memory { @@ -1711,10 +1749,11 @@ impl SqliteStore { run_id: &str, task_id: &str, metadata: &TaskExecutionMetadata, + execution_memory: &Value, now: DateTime, ) -> Result<(), StoreError> { let changed = self.connection.lock().execute( - "UPDATE task_states SET metadata_version = ?3, definition_fingerprint = ?4, input_digest = ?5, output_contract_fingerprint = ?6, updated_at = ?7 WHERE run_id = ?1 AND task_id = ?2 AND state = ?8", + "UPDATE task_states SET metadata_version = ?3, definition_fingerprint = ?4, input_digest = ?5, output_contract_fingerprint = ?6, execution_memory_json = ?7, updated_at = ?8 WHERE run_id = ?1 AND task_id = ?2 AND state = ?9", params![ run_id, task_id, @@ -1722,6 +1761,11 @@ impl SqliteStore { metadata.definition_fingerprint, metadata.input_digest, metadata.output_contract_fingerprint, + encode_protected( + &self.protection, + execution_memory, + "task_states.execution_memory_json" + )?, now.to_rfc3339(), encode_enum(TaskState::Running)?, ], @@ -1744,94 +1788,294 @@ impl SqliteStore { metadata: &TaskCompletionMetadata, now: DateTime, trace_id: &str, + ) -> Result<(), StoreError> { + self.commit_task_batch( + run_id, + &[TaskBatchResult { + task_id: task_id.to_owned(), + outcome: TaskBatchOutcome::Succeeded { + output: output.clone(), + metadata: Box::new(metadata.clone()), + }, + }], + working_memory, + false, + now, + trace_id, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn commit_task_batch( + &self, + run_id: &str, + results: &[TaskBatchResult], + working_memory: Option<&Value>, + fail_run: bool, + now: DateTime, + trace_id: &str, ) -> Result<(), StoreError> { let _artifact_guard = self.artifact_lock.lock(); let _artifact_file_lock = self.artifact_store.lock_exclusive()?; - verify_artifact_manifest(self.artifact_store.as_ref(), &metadata.artifact_manifest)?; + for result in results { + if let TaskBatchOutcome::Succeeded { metadata, .. } = &result.outcome { + verify_artifact_manifest( + self.artifact_store.as_ref(), + &metadata.artifact_manifest, + )?; + } + } let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let current: String = transaction - .query_row( - "SELECT state FROM task_states WHERE run_id = ?1 AND task_id = ?2", - params![run_id, task_id], - |row| row.get(0), - ) - .optional()? - .ok_or_else(|| StoreError::TaskNotFound { - run_id: run_id.to_owned(), - task_id: task_id.to_owned(), - })?; - let current: TaskState = decode_enum(¤t, "task.state")?; - current - .transition(TaskState::Succeeded) - .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; - transaction.execute( - "UPDATE task_states SET state = ?3, output_json = ?4, error = NULL, disposition = ?5, metadata_version = ?6, definition_fingerprint = ?7, input_digest = ?8, output_contract_fingerprint = ?9, output_digest = ?10, state_delta_json = ?11, state_delta_digest = ?12, artifact_manifest_json = ?13, updated_at = ?14 WHERE run_id = ?1 AND task_id = ?2", - params![ - run_id, - task_id, - encode_enum(TaskState::Succeeded)?, - encode_protected(&self.protection, output, "task_states.output_json")?, - encode_enum(TaskDisposition::Executed)?, - metadata.execution.metadata_version, - metadata.execution.definition_fingerprint, - metadata.execution.input_digest, - metadata.execution.output_contract_fingerprint, - metadata.output_digest, - encode_protected( + for result in results { + let current: String = transaction + .query_row( + "SELECT state FROM task_states WHERE run_id = ?1 AND task_id = ?2", + params![run_id, result.task_id], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| StoreError::TaskNotFound { + run_id: run_id.to_owned(), + task_id: result.task_id.clone(), + })?; + let current: TaskState = decode_enum(¤t, "task.state")?; + let next = match &result.outcome { + TaskBatchOutcome::Succeeded { .. } => TaskState::Succeeded, + TaskBatchOutcome::Failed { .. } => TaskState::Failed, + TaskBatchOutcome::RetryScheduled { .. } => TaskState::RetryScheduled, + }; + current + .transition(next) + .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; + match &result.outcome { + TaskBatchOutcome::Succeeded { output, metadata } => { + transaction.execute( + "UPDATE task_states SET state = ?3, output_json = ?4, error = NULL, disposition = ?5, metadata_version = ?6, definition_fingerprint = ?7, input_digest = ?8, output_contract_fingerprint = ?9, output_digest = ?10, state_delta_json = ?11, state_delta_digest = ?12, artifact_manifest_json = ?13, updated_at = ?14 WHERE run_id = ?1 AND task_id = ?2", + params![ + run_id, + result.task_id, + encode_enum(TaskState::Succeeded)?, + encode_protected( + &self.protection, + output, + "task_states.output_json" + )?, + encode_enum(TaskDisposition::Executed)?, + metadata.execution.metadata_version, + metadata.execution.definition_fingerprint, + metadata.execution.input_digest, + metadata.execution.output_contract_fingerprint, + metadata.output_digest, + encode_protected( + &self.protection, + &metadata.state_delta, + "task_states.state_delta_json" + )?, + metadata.state_delta_digest, + encode(&metadata.artifact_manifest)?, + now.to_rfc3339(), + ], + )?; + record_artifact_references_tx( + &transaction, + run_id, + &result.task_id, + &metadata.artifact_manifest, + None, + None, + now, + )?; + transaction.execute( + "DELETE FROM artifact_ingests WHERE run_id = ?1 AND task_id = ?2", + params![run_id, result.task_id], + )?; + append_audit_tx( + &transaction, + run_id, + "task.transition", + Some(&result.task_id), + trace_id, + &serde_json::json!({ + "from": current, + "to": TaskState::Succeeded, + "disposition": TaskDisposition::Executed, + "outputDigest": metadata.output_digest, + "stateDeltaDigest": metadata.state_delta_digest, + }), + now, + &self.protection, + )?; + } + TaskBatchOutcome::Failed { error } | TaskBatchOutcome::RetryScheduled { error } => { + transaction.execute( + "UPDATE task_states SET state = ?3, error = ?4, updated_at = ?5 WHERE run_id = ?1 AND task_id = ?2", + params![ + run_id, + result.task_id, + encode_enum(next)?, + protect_text(&self.protection, error, "task_states.error")?, + now.to_rfc3339(), + ], + )?; + append_audit_tx( + &transaction, + run_id, + "task.transition", + Some(&result.task_id), + trace_id, + &serde_json::json!({"from": current, "to": next, "error": error}), + now, + &self.protection, + )?; + } + } + } + + if fail_run { + let mut statement = transaction.prepare( + "SELECT task_id, state FROM task_states WHERE run_id = ?1 ORDER BY position", + )?; + let remaining = statement + .query_map([run_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + drop(statement); + for (task_id, encoded) in remaining { + let current: TaskState = decode_enum(&encoded, "task.state")?; + if current.is_terminal() { + continue; + } + current + .transition(TaskState::Cancelled) + .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; + transaction.execute( + "UPDATE task_states SET state = ?3, error = ?4, updated_at = ?5 WHERE run_id = ?1 AND task_id = ?2", + params![ + run_id, + task_id, + encode_enum(TaskState::Cancelled)?, + protect_text( + &self.protection, + "cancelled after a stop-on-failure task failed", + "task_states.error" + )?, + now.to_rfc3339(), + ], + )?; + append_audit_tx( + &transaction, + run_id, + "task.transition", + Some(&task_id), + trace_id, + &serde_json::json!({ + "from": current, + "to": TaskState::Cancelled, + "error": "cancelled after a stop-on-failure task failed", + }), + now, &self.protection, - &metadata.state_delta, - "task_states.state_delta_json" - )?, - metadata.state_delta_digest, - encode(&metadata.artifact_manifest)?, - now.to_rfc3339(), - ], - )?; - record_artifact_references_tx( - &transaction, - run_id, - task_id, - &metadata.artifact_manifest, - None, - None, - now, - )?; - transaction.execute( - "DELETE FROM artifact_ingests WHERE run_id = ?1 AND task_id = ?2", - params![run_id, task_id], - )?; - if let Some(memory) = working_memory { + )?; + } transaction.execute( - "UPDATE runs SET working_memory_json = ?2, updated_at = ?3 WHERE run_id = ?1", + "UPDATE effects SET status = ?2, error = ?3, completed_at = ?4, confirmed = 0 WHERE run_id = ?1 AND status IN (?5, ?6)", params![ run_id, - encode_protected(&self.protection, memory, "runs.working_memory_json")?, - now.to_rfc3339() + encode_enum(EffectStatus::Cancelled)?, + protect_text( + &self.protection, + "cancelled after a stop-on-failure task failed", + "effects.error" + )?, + now.to_rfc3339(), + encode_enum(EffectStatus::Requested)?, + encode_enum(EffectStatus::WaitingForApproval)?, ], )?; - } else { transaction.execute( - "UPDATE runs SET updated_at = ?2 WHERE run_id = ?1", - params![run_id, now.to_rfc3339()], + "UPDATE approvals SET status = 'cancelled', resolved_at = ?2, resolved_by = 'runtime', resolution_reason = ?3 WHERE run_id = ?1 AND status = 'pending'", + params![ + run_id, + now.to_rfc3339(), + protect_text( + &self.protection, + "run stopped after task failure", + "approvals.resolution_reason" + )?, + ], + )?; + } + + let current_run_state = if fail_run { + let encoded: String = transaction + .query_row( + "SELECT state FROM runs WHERE run_id = ?1", + [run_id], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| StoreError::RunNotFound(run_id.to_owned()))?; + let current: RunState = decode_enum(&encoded, "run.state")?; + current + .transition(RunState::Failed) + .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; + Some(current) + } else { + None + }; + match (working_memory, fail_run) { + (Some(memory), true) => { + transaction.execute( + "UPDATE runs SET working_memory_json = ?2, state = ?3, updated_at = ?4 WHERE run_id = ?1", + params![ + run_id, + encode_protected( + &self.protection, + memory, + "runs.working_memory_json" + )?, + encode_enum(RunState::Failed)?, + now.to_rfc3339() + ], + )?; + } + (Some(memory), false) => { + transaction.execute( + "UPDATE runs SET working_memory_json = ?2, updated_at = ?3 WHERE run_id = ?1", + params![ + run_id, + encode_protected(&self.protection, memory, "runs.working_memory_json")?, + now.to_rfc3339() + ], + )?; + } + (None, true) => { + transaction.execute( + "UPDATE runs SET state = ?2, updated_at = ?3 WHERE run_id = ?1", + params![run_id, encode_enum(RunState::Failed)?, now.to_rfc3339()], + )?; + } + (None, false) => { + transaction.execute( + "UPDATE runs SET updated_at = ?2 WHERE run_id = ?1", + params![run_id, now.to_rfc3339()], + )?; + } + } + if let Some(current) = current_run_state { + append_audit_tx( + &transaction, + run_id, + "run.state", + None, + trace_id, + &serde_json::json!({"from": current, "to": RunState::Failed}), + now, + &self.protection, )?; } - append_audit_tx( - &transaction, - run_id, - "task.transition", - Some(task_id), - trace_id, - &serde_json::json!({ - "from": current, - "to": TaskState::Succeeded, - "disposition": TaskDisposition::Executed, - "outputDigest": metadata.output_digest, - "stateDeltaDigest": metadata.state_delta_digest, - }), - now, - &self.protection, - )?; checkpoint_tx(&transaction, run_id, now, &self.protection)?; transaction.commit()?; Ok(()) @@ -2098,45 +2342,10 @@ impl SqliteStore { ) -> Result { let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - transaction.execute( - "INSERT INTO effects (effect_id, format_version, run_id, task_id, task_attempt, ordinal, operation, effect_class, risk, idempotency, idempotency_key, input_digest, input_json, expected_effect, trace_id, status, effect_attempt, requested_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, 1, ?17)", - params![ - request.id, - request.format_version, - request.run_id, - request.task_id, - request.attempt, - request.ordinal, - request.operation, - encode_enum(request.effect_class)?, - encode_enum(request.risk)?, - encode_enum(request.idempotency)?, - request.idempotency_key, - request.input_digest, - encode_protected(&self.protection, &request.input, "effects.input_json")?, - protect_text( - &self.protection, - &request.expected_effect, - "effects.expected_effect" - )?, - request.trace_id, - encode_enum(EffectStatus::Requested)?, - now.to_rfc3339(), - ], - )?; - append_audit_tx( + insert_effect_request_tx( &transaction, - &request.run_id, - "effect.requested", - Some(&request.task_id), - &request.trace_id, - &serde_json::json!({ - "effectId": request.id, - "operation": request.operation, - "inputDigest": request.input_digest, - "effectClass": request.effect_class, - "risk": request.risk, - }), + request, + EffectStatus::Requested, now, &self.protection, )?; @@ -2653,9 +2862,35 @@ impl SqliteStore { effect_id.map(|id| self.load_effect(&id)).transpose() } - pub fn create_approval(&self, request: &ApprovalRequest) -> Result<(), StoreError> { + pub fn create_approval( + &self, + effect: &EffectRequest, + request: &ApprovalRequest, + ) -> Result<(), StoreError> { let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current: String = transaction + .query_row( + "SELECT state FROM task_states WHERE run_id = ?1 AND task_id = ?2", + params![request.run_id, request.task_id], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| StoreError::TaskNotFound { + run_id: request.run_id.clone(), + task_id: request.task_id.clone(), + })?; + let current: TaskState = decode_enum(¤t, "task.state")?; + current + .transition(TaskState::WaitingForApproval) + .map_err(|transition| StoreError::InvalidTransition(transition.to_string()))?; + insert_effect_request_tx( + &transaction, + effect, + EffectStatus::WaitingForApproval, + request.requested_at, + &self.protection, + )?; transaction.execute( "INSERT INTO approvals (approval_id, run_id, effect_id, task_id, agent, tool, capability, risk, redacted_input_json, expected_effect, reason, trace_id, status, requested_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'pending', ?13)", params![ @@ -2683,12 +2918,33 @@ impl SqliteStore { ], )?; transaction.execute( - "UPDATE effects SET status = ?2 WHERE effect_id = ?1", + "UPDATE task_states SET state = ?3, error = NULL, updated_at = ?4 WHERE run_id = ?1 AND task_id = ?2", params![ - request.effect_id, - encode_enum(EffectStatus::WaitingForApproval)? + request.run_id, + request.task_id, + encode_enum(TaskState::WaitingForApproval)?, + request.requested_at.to_rfc3339(), ], )?; + append_audit_tx( + &transaction, + &request.run_id, + "task.transition", + Some(&request.task_id), + &request.trace_id, + &serde_json::json!({ + "from": current, + "to": TaskState::WaitingForApproval, + }), + request.requested_at, + &self.protection, + )?; + checkpoint_tx( + &transaction, + &request.run_id, + request.requested_at, + &self.protection, + )?; transaction.commit()?; Ok(()) } @@ -3753,6 +4009,7 @@ fn migrate(connection: &mut Connection) -> Result<(), StoreError> { (8_u32, MIGRATION_8), (9_u32, MIGRATION_9), (10_u32, MIGRATION_10), + (11_u32, MIGRATION_11), ]; for (version, sql) in migrations .into_iter() @@ -3911,6 +4168,58 @@ fn append_audit_tx( Ok(()) } +fn insert_effect_request_tx( + transaction: &Transaction<'_>, + request: &EffectRequest, + status: EffectStatus, + now: DateTime, + protection: &SharedStateProtection, +) -> Result<(), StoreError> { + transaction.execute( + "INSERT INTO effects (effect_id, format_version, run_id, task_id, task_attempt, ordinal, operation, effect_class, risk, idempotency, idempotency_key, input_digest, input_json, expected_effect, trace_id, status, effect_attempt, requested_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, 1, ?17)", + params![ + request.id, + request.format_version, + request.run_id, + request.task_id, + request.attempt, + request.ordinal, + request.operation, + encode_enum(request.effect_class)?, + encode_enum(request.risk)?, + encode_enum(request.idempotency)?, + request.idempotency_key, + request.input_digest, + encode_protected(protection, &request.input, "effects.input_json")?, + protect_text( + protection, + &request.expected_effect, + "effects.expected_effect" + )?, + request.trace_id, + encode_enum(status)?, + now.to_rfc3339(), + ], + )?; + append_audit_tx( + transaction, + &request.run_id, + "effect.requested", + Some(&request.task_id), + &request.trace_id, + &serde_json::json!({ + "effectId": request.id, + "operation": request.operation, + "inputDigest": request.input_digest, + "effectClass": request.effect_class, + "risk": request.risk, + "status": status, + }), + now, + protection, + ) +} + fn append_trace_tx( transaction: &Transaction<'_>, run_id: &str, @@ -4381,6 +4690,7 @@ spec: MIGRATION_7, MIGRATION_8, MIGRATION_9, + MIGRATION_10, ] .into_iter() .enumerate() @@ -4754,6 +5064,61 @@ spec: assert_eq!(store.checkpoint_count("run").expect("count"), 4); } + #[test] + fn parallel_batch_commit_rolls_back_every_task_and_memory_on_failure() { + let store = SqliteStore::open_memory().expect("store"); + create(&store, "run"); + begin_task(&store, "run"); + let metadata = TaskCompletionMetadata { + execution: TaskExecutionMetadata { + metadata_version: 1, + definition_fingerprint: "definition".to_owned(), + input_digest: "input".to_owned(), + output_contract_fingerprint: "contract".to_owned(), + }, + output_digest: "output".to_owned(), + state_delta: serde_json::json!({ + "formatVersion": 1, + "set": {"value": "changed"}, + "remove": [], + }), + state_delta_digest: "delta".to_owned(), + artifact_manifest: Vec::new(), + }; + let result = store.commit_task_batch( + "run", + &[ + TaskBatchResult { + task_id: "one".to_owned(), + outcome: TaskBatchOutcome::Succeeded { + output: serde_json::json!({"ok": true}), + metadata: Box::new(metadata.clone()), + }, + }, + TaskBatchResult { + task_id: "missing".to_owned(), + outcome: TaskBatchOutcome::Succeeded { + output: serde_json::json!({"ok": true}), + metadata: Box::new(metadata), + }, + }, + ], + Some(&serde_json::json!({"value": "changed"})), + false, + Utc::now(), + "trace", + ); + assert!(matches!(result, Err(StoreError::TaskNotFound { .. }))); + assert_eq!( + store.list_tasks("run").expect("tasks")[0].state, + TaskState::Running + ); + assert_eq!( + store.load_run("run").expect("run").working_memory, + serde_json::json!({}) + ); + } + #[test] fn corrupt_rows_fail_without_panicking() { let store = SqliteStore::open_memory().expect("store"); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6cf8a90..3d30299 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -11,7 +11,7 @@ CLI ───────┬────────> runtime ─────> c runtime ────────────> observability ─> core contracts ``` -`agentctl-core` owns deterministic domain behavior: strict parsing, migration, compilation, template resolution, effect identities, state machines, policy, and provider/tool interfaces. It knows no HTTP client, database, or CLI type. `agentctl-store` is the SQLite implementation. `agentctl-runtime` schedules one stable ready task at a time and coordinates injected clocks, IDs, executors, providers, protocols, persistence, and traces. Concrete network adapters and rendering stay at the edges. +`agentctl-core` owns deterministic domain behavior: strict parsing, migration, compilation, template resolution, effect identities, state machines, policy, and provider/tool interfaces. It knows no HTTP client, database, or CLI type. `agentctl-store` is the SQLite implementation. `agentctl-runtime` schedules stable bounded ready batches and coordinates injected clocks, IDs, executors, providers, protocols, persistence, and traces. Concrete network adapters and rendering stay at the edges. ## Execution @@ -23,7 +23,19 @@ State transitions, checkpoint creation, working-memory replacement, artifact ref ## Determinism and concurrency -Ready tasks are ordered by YAML declaration order after dependencies. `maxConcurrency` currently must be `1`. Parallel execution, loops, matrix/foreach expansion, routers, sub-workflows, handlers, compensation execution, and event triggers are deferred because deterministic merge and recovery semantics are not yet frozen. The DSL carries optional compensation metadata on a tool contract, but the runtime does not execute compensation. +Ready tasks are ordered by YAML declaration order after dependencies. +`maxConcurrency` defaults to one and is bounded at 64. A parallel batch reads +per-task durable memory snapshots, executes independently, then commits +successful outputs, disjoint memory deltas, failures, artifact references, +audit events, and the checkpoint in compiled order in one transaction. +Unordered overlapping `memoryWrites` fail compilation. Effects and provider +sessions remain task-local. See ADR 0008 and +[Deterministic parallel tasks](guides/PARALLEL_TASKS.md). + +Loops, matrix/foreach expansion, routers, sub-workflows, handlers, +compensation execution, and event triggers still require their own explicit +state and recovery contracts. The DSL carries optional compensation metadata +on a tool contract, but the runtime does not execute compensation. Clock and identifier generation are injected. Provider responses, tools, and external actions are injected interfaces. Cryptographic digests canonicalize identity; output maps use stable ordering where the public contract requires it. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 96d0085..ec34653 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -22,4 +22,4 @@ Legacy workflows depending on packs, broad built-in tool profiles, remote MCP/A2 ## Deferred product decisions -Parallel execution, foreach/matrix, loops, routers, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. +Foreach/matrix, loops, routers, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. diff --git a/docs/DSL.md b/docs/DSL.md index efd1f8f..c24e120 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,7 +2,7 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. @@ -17,10 +17,10 @@ trace, or inspection output. See [Secret references](guides/SECRET_REFERENCES.md `policy.workspaceRoot` is the default boundary for relative file paths. Each `writableRoots` entry may be workspace-relative or an explicit absolute mount such as `/artifacts`. Ordinary reads remain workspace-confined. After a successful authorized mutation, the runtime may read that exact output through its writable-root boundary to ingest the bounded regular file into durable CAS; this does not grant tasks general read access to the external root. -The compiler validates missing references, duplicate tasks, cycles, task-aware templates, tool references, provider capabilities, agent limits, and sequential runtime settings before execution. Ready tasks follow declaration order. `maxConcurrency` must be `1` in this version. +The compiler validates missing references, duplicate tasks, cycles, task-aware templates, tool references, provider capabilities, agent limits, concurrency bounds, and working-memory conflicts before execution. `maxConcurrency` accepts `1` through `64` and defaults to `1`. Independent ready tasks are selected in compiled order, execute against durable isolated memory snapshots, and commit atomically in compiled order. Literal working-memory keys are inferred; templated keys require `memoryWrites`. See [Deterministic parallel tasks](guides/PARALLEL_TASKS.md). `builtin.shell.exec` captures stdout and stderr concurrently. Its optional `stdoutLimitBytes`, `stderrLimitBytes`, and `combinedOutputLimitBytes` fields default to 1 MiB, 1 MiB, and 2 MiB respectively. Each configured value must be between 1 byte and 16 MiB. `timeoutSeconds` must be between 1 and 86,400. Exceeding an output bound terminates and reaps the process and records a structured failed effect; timeout or cancellation remains an uncertain effect because external changes may already have occurred. These fields are validated identically for workflow and pack actions. The parser translates a limited unversioned `playbook:` document and emits a migration warning. Use `agentctl migrate old.yaml --write new.yaml`. Legacy pack-backed, MCP, A2A, provider-specific, and broad module configurations need manual migration; see [Migrating from TypeScript](MIGRATING_FROM_TYPESCRIPT.md). -Not implemented in v1alpha1: `foreach`, matrix expansion, parallel groups, routers, loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. They remain excluded until their deterministic state, merge, and recovery semantics are specified. +Not implemented in v1alpha1: `foreach`, matrix expansion, routers, loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index c64118a..9d80108 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -25,6 +25,10 @@ Cancellation is both an injected token and a durable run flag. CLI SIGINT and SI A repaired agent task starts a fresh provider session. Source `previous_response_id`, incomplete turns, pending tool calls, and reasoning state are not copied. Validated task output and reconstructed memory are the only cross-task/cross-run dataflow. -Clock and ID generation are injected; test providers/tools/protocol handlers are injected. The current scheduler is sequential, so output and memory commit order is task declaration order. +Clock and ID generation are injected; test providers/tools/protocol handlers +are injected. Ready tasks execute in bounded stable batches. Each reads a +persisted immutable memory snapshot, while task output, disjoint memory deltas, +artifacts, failures, audit events, and the checkpoint commit atomically in +compiled order. The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 94a0b6c..9f3458e 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -23,7 +23,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an These are useful extensions but are not required by the product thesis. They need new deterministic state and compatibility contracts before implementation: -- parallel task execution; `foreach` and matrix expansion; loops; routers; sub-workflows; compensation execution; +- `foreach` and matrix expansion; loops; routers; sub-workflows; compensation execution; - structured agent teams and handoffs; - model token streaming into CLI/workflow state; - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; @@ -42,7 +42,7 @@ These are useful extensions but are not required by the product thesis. They nee ## Current operational limits - The document API is `v1alpha1`; pin the binary/image version and validate before upgrading. -- Scheduling is sequential (`maxConcurrency: 1`). Separate runs may overlap safely in SQLite, but they can still target the same external resource. Use the external scheduler's overlap controls (`flock`, systemd unit serialization, or Kubernetes `concurrencyPolicy: Forbid`) when effects must not overlap. +- Parallel scheduling is local to one run and process, bounded at 64 tasks, and defaults to sequential execution. Working-memory conflicts fail compilation, but tasks that target the same external resource still require explicit `needs` ordering or that system's concurrency controls. Separate runs also require external overlap controls when effects must not overlap. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 0f2a81f..24df1ed 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -34,6 +34,6 @@ cancellation. See [Secret references](guides/SECRET_REFERENCES.md). `cargo xtask examples-verify-live-openai` is the broader opt-in gate. It runs every public OpenAI workflow plus the canonical two-agent repair. The repaired task starts a new Responses session and uses `previous_response_id` only between its own new turns. The failed source task's response ID, pending tool call, and reasoning state are not copied. Validated task output is the cross-run dataflow boundary. -OpenAI provider options are an allowlisted map (`store`, `reasoningContext`, `promptCacheMode`, `promptCacheTtl`, `parallelToolCalls`, and `safetyIdentifier`). Unknown options or invalid values fail compilation. Tool-using OpenAI and Azure OpenAI agents may not set `store: false`: stateless continuation would require replaying returned response/reasoning/function items, which this release does not implement. One-turn agents without tools may disable storage. Programmatic tool calling and model streaming are explicitly unsupported in the workflow runtime and fail rather than being ignored. Parallel function calls are parsed and correlated, but executors run them serially in response order because v1 scheduling is sequential. +OpenAI provider options are an allowlisted map (`store`, `reasoningContext`, `promptCacheMode`, `promptCacheTtl`, `parallelToolCalls`, and `safetyIdentifier`). Unknown options or invalid values fail compilation. Tool-using OpenAI and Azure OpenAI agents may not set `store: false`: stateless continuation would require replaying returned response/reasoning/function items, which this release does not implement. One-turn agents without tools may disable storage. Programmatic tool calling and model streaming are explicitly unsupported in the workflow runtime and fail rather than being ignored. Parallel function calls are parsed and correlated, but one agent task executes them serially in response order. Independent workflow tasks can use bounded parallel scheduling. Cost is not inferred when a provider returns no reliable cost metadata. A workflow requesting `maxCostUsd` therefore fails capability negotiation; input/output token limits are enforced from native usage. Retry is limited to explicit task bounds and definitive retryable HTTP responses. Timeout, cancellation, or a transport loss after dispatch is considered ambiguous and is not automatically reissued. diff --git a/docs/adr/0005-narrow-v1-scheduling-and-extensions.md b/docs/adr/0005-narrow-v1-scheduling-and-extensions.md index c256359..356be54 100644 --- a/docs/adr/0005-narrow-v1-scheduling-and-extensions.md +++ b/docs/adr/0005-narrow-v1-scheduling-and-extensions.md @@ -1,6 +1,6 @@ # ADR 0005: Narrow v1 scheduling and extension surface -Status: accepted, 2026-07-22. +Status: superseded for scheduling by ADR 0008, 2026-07-24. V1alpha1 schedules a sequential DAG in declaration order and integrates typed actions/tools, local packs, MCP 2025-11-25, and A2A 1.0. `maxConcurrency` greater than one is rejected. diff --git a/docs/adr/0008-deterministic-parallel-batches.md b/docs/adr/0008-deterministic-parallel-batches.md new file mode 100644 index 0000000..edeff0e --- /dev/null +++ b/docs/adr/0008-deterministic-parallel-batches.md @@ -0,0 +1,20 @@ +# ADR 0008: Deterministic parallel batches + +Status: accepted, 2026-07-24. + +Independent ready tasks may execute concurrently when +`runtime.maxConcurrency` is greater than one. Selection follows compiled plan +order and never exceeds 64 tasks. + +Each task receives a durable immutable working-memory snapshot. Completed +results commit in compiled order as one SQLite transaction, including task +states, working-memory deltas, artifact references, audit events, run failure +when applicable, and the checkpoint. + +Working-memory write sets are part of the compiled task. Literal +`builtin.memory.write` keys are inferred, templated keys require +`memoryWrites`, and unordered overlaps fail compilation. No implicit merge +strategy exists. + +This replaces the sequential-only scheduling decision in ADR 0005. The default +concurrency remains one for compatibility. diff --git a/docs/architecture/DIAGRAMS.md b/docs/architecture/DIAGRAMS.md index 6d408f7..8ddef3f 100644 --- a/docs/architecture/DIAGRAMS.md +++ b/docs/architecture/DIAGRAMS.md @@ -12,7 +12,7 @@ flowchart LR accDescr: A workflow author uses the CLI, which joins deterministic core contracts to runtime providers, protocols, executors, tracing, and SQLite. User[Workflow author or operator] --> CLI[agentctl CLI] CLI --> Core[Core parser compiler policy state] - CLI --> Runtime[Sequential runtime] + CLI --> Runtime[Bounded deterministic runtime] Runtime --> Store[SQLite store] Runtime --> Providers[Native model providers] Runtime --> Protocols[MCP and A2A clients] @@ -48,9 +48,9 @@ A normal run moves through durable states and ends in one terminal result or a d flowchart TD accTitle: Run lifecycle accDescr: A run creates durable records, executes ready tasks, persists results, and reaches success, approval, failure, or cancellation. - Create[Create run and task records] --> Ready[Find next ready task] - Ready --> Execute[Execute action or bounded agent] - Execute --> Persist[Commit task output checkpoint and audit] + Create[Create run and task records] --> Ready[Select stable ready batch] + Ready --> Execute[Execute independent tasks] + Execute --> Persist[Commit ordered batch checkpoint and audit] Persist --> More{More ready tasks?} More -->|Yes| Ready More -->|No| Success[Succeeded] @@ -58,7 +58,9 @@ flowchart TD Execute --> Failure[Failed or cancelled] ``` -The current scheduler runs one ready task at a time in declaration order. A pending approval is non-terminal and can later resume. +The scheduler selects up to `maxConcurrency` ready tasks in declaration order. +Every task reads a durable isolated snapshot. Results commit atomically in plan +order. A pending approval is non-terminal and can later resume. ## Runtime state machine diff --git a/docs/development/REPOSITORY.md b/docs/development/REPOSITORY.md index 4504979..44ba712 100644 --- a/docs/development/REPOSITORY.md +++ b/docs/development/REPOSITORY.md @@ -8,7 +8,7 @@ The production implementation is a Rust workspace. The remaining top-level TypeS | --- | --- | | `agentctl-core` | strict DSL, migration, compiler, templates, state, effects, policy, provider and tool contracts | | `agentctl-store` | versioned SQLite persistence, migrations, checkpoints, approvals, audit, trace, sessions, tool calls, memory | -| `agentctl-runtime` | sequential scheduler, actions, bounded agent loop, resume, replay, fork, cancellation | +| `agentctl-runtime` | bounded deterministic scheduler, actions, bounded agent loop, resume, replay, fork, cancellation | | `agentctl-providers` | native OpenAI, Azure OpenAI, Anthropic, Google, and fake adapters | | `agentctl-protocols` | MCP and A2A clients | | `agentctl-observability` | typed events, test sink, and OpenTelemetry bridge | diff --git a/docs/execution/DECISIONS.md b/docs/execution/DECISIONS.md index cea8e70..54221a1 100644 --- a/docs/execution/DECISIONS.md +++ b/docs/execution/DECISIONS.md @@ -6,8 +6,9 @@ | [0002](../adr/0002-versioned-strict-workflow-envelope.md) | Versioned strict envelope | accepted | Generated schema, strict fields, explicit migration. | | [0003](../adr/0003-sqlite-history-and-conservative-recovery.md) | SQLite history, conservative recovery | accepted | Resume reuses confirmed effects; replay performs none; fork is fresh. | | [0004](../adr/0004-native-provider-adapters.md) | Native providers behind neutral contracts | accepted | Native request/response tests and early capability rejection. | -| [0005](../adr/0005-narrow-v1-scheduling-and-extensions.md) | Sequential v1 and narrow extensions | accepted | Deterministic commits now; parallel/dynamic constructs deferred. | +| [0005](../adr/0005-narrow-v1-scheduling-and-extensions.md) | Narrow v1 scheduling and extensions | superseded for scheduling | Dynamic constructs remain separate decisions; ADR 0008 replaces the sequential-only rule. | | [0006](../adr/0006-schedulable-runtime-and-noninteractive-contract.md) | Schedulable runtime, durable non-interactive pause | accepted | External platforms schedule; CLI never prompts or auto-approves in CI. | | [0007](../adr/0007-generic-oci-step-contract.md) | Generic OCI step contract | accepted | Non-root/read-only image; mounted config/workspace/state/artifacts. | +| [0008](../adr/0008-deterministic-parallel-batches.md) | Deterministic parallel batches | accepted | Bounded overlap, isolated snapshots, declared writes, and atomic plan-order commits. | These decisions resolve the researched patterns in [LANDSCAPE.md](../research/LANDSCAPE.md). No unsafe code or distributed control plane ADR is required because neither exists. diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md index dbc110f..a5d5290 100644 --- a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -40,6 +40,7 @@ This inventory is enforced by `cargo xtask examples-verify`. The default command | `examples/v1/long-term-memory.yaml` | Namespaced durable memory | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | | `examples/v1/mcp.yaml` | MCP call contract | MCP | external execution | 0 | 0 | Protocol mock | Protocol mock | N/A | N/A | N/A | Static | passed | | `examples/v1/openai-live.yaml` | Minimal OpenAI response | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Live gate | live passed | +| `examples/v1/parallel.yaml` | Deterministic parallel batch | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Atomic ordered memory merge | Canonical | passed | | `examples/v1/policy-denial.yaml` | Denied mutation | deterministic | policy failure | 0 | 0 | Canonical expected failure | N/A | N/A | N/A | No mutation | JSON error | passed | | `examples/v1/reusable-pack.yaml` | Native reusable pack consumer | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Pack digest | Canonical | passed | | `examples/v1/secret-reference.yaml` | Environment reference contract | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Secret-safe live gate | live passed | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index 99ea3d7..e5d84a1 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -43,7 +43,7 @@ complete, every entry must have exactly one final disposition: | NET-001 | Network policy | open | implemented | | ISO-001 | Process isolation | open | redesigned | | BUD-001 | Resource and cost budgets | open | implemented | -| SCH-001 | Deterministic parallel execution | open | implemented | +| SCH-001 | Deterministic parallel execution | in progress | implemented | | DYN-001 | Foreach and matrix | open | implemented | | COND-001 | Conditions and routers | open | implemented | | LOOP-001 | Bounded loops | open | implemented | @@ -207,7 +207,7 @@ complete, every entry must have exactly one final disposition: ### SCH-001: Deterministic parallel scheduling -- Current behavior: `maxConcurrency` must be 1. +- Current behavior: `maxConcurrency` accepts 1 through 64 and defaults to 1. - User impact: independent model and deterministic tasks cannot overlap. - Security or durability impact: naive concurrency would make state and effect order race-dependent. @@ -217,13 +217,21 @@ complete, every entry must have exactly one final disposition: - Required implementation: concurrency semaphore, isolated task snapshots, ordered commit queue, failure/cancellation/approval behavior, effect and trace parentage, repair/retry/replay integration, and plan visibility. -- Migration impact: runtime/plan format versions and DSL schema change. +- Migration impact: additive DSL/plan fields retain sequential defaults. + Database migration 11 adds an encrypted-capable per-task execution-memory + snapshot so crashes and approvals preserve the original batch boundary. - Tests: real overlap, stable commit order, conflict rejection, cancellation, approval, failures, effects, replay, repair, and container execution. - Examples: parallel deterministic and agent branches. - Live evidence: two bounded OpenAI branches. - Documentation: scheduling and state conflict rules. -- Final disposition: pending implementation evidence. +- Final disposition: implemented. Deterministic verification covers real + overlap and the concurrency cap, compiled write-conflict rejection, runtime + declared-key enforcement before effects, atomic rollback injection, + plan-order audit assertions, disjoint state merge, stop/continue boundary + behavior, cancellation, multi-approval resume, failed-only retry, selective + repair, and offline replay. Program state remains in progress until the + bounded live OpenAI branch scenario and OCI gate execute. ### DYN-001: Bounded foreach and matrix expansion diff --git a/docs/execution/RELEASE_AUDIT.md b/docs/execution/RELEASE_AUDIT.md index ed6e888..ee9069c 100644 --- a/docs/execution/RELEASE_AUDIT.md +++ b/docs/execution/RELEASE_AUDIT.md @@ -222,7 +222,7 @@ Coverage percentage was not invented; `cargo-llvm-cov` was unavailable. Timing-s - Dispatch the configured Linux amd64, macOS, Windows, scan/SBOM, and external pipeline gates; until then they remain CI-configured or documentation-reviewed only. - Expand Azure/Anthropic/Google adapter negative/error/cancellation/tool-continuation coverage before raising their maturity beyond focused mock mapping. -- Single-host SQLite, sequential scheduling, manual uncertain-effect reconciliation, alpha schema evolution, and policy-not-sandbox limitations remain intentional. +- Single-host SQLite, local bounded parallel scheduling, explicit uncertain-effect reconciliation, alpha schema evolution, and policy-not-sandbox limitations remain intentional. - Tool-using OpenAI/Azure `store: false` remains unsupported until full stateless response-item replay is implemented. - Container bind mounts require deliberate UID/GID 65532 provisioning and protected collection of state, which may contain prompts and outputs. - Formal external MCP/A2A/provider conformance suites and long-horizon upgrade fixtures are deferred. diff --git a/docs/execution/STATUS.md b/docs/execution/STATUS.md index f63c4a5..dc6020e 100644 --- a/docs/execution/STATUS.md +++ b/docs/execution/STATUS.md @@ -22,7 +22,13 @@ The independent release-candidate review found and remediated one P0, five P1s, ## Product boundary -`agentctl` is a schedulable local runtime, not a scheduler or distributed control plane. The workflow API remains alpha and scheduling is sequential. Provider, filesystem, process, and network policy is not an OS sandbox. At-most-once external work can require manual reconciliation. See [Limitations](../LIMITATIONS.md) for the complete release-blocker/hardening/post-v1/non-goal classification. +`agentctl` is a schedulable local runtime, not an external scheduler or +distributed control plane. The workflow API remains alpha. One run may execute +bounded independent tasks concurrently with deterministic plan-order commits; +cross-run and cross-host overlap remains external. Provider, filesystem, +process, and network policy is not an OS sandbox. At-most-once external work +can require explicit reconciliation. See [Limitations](../LIMITATIONS.md) for +the complete release-blocker/hardening/post-v1/non-goal classification. ## External evidence not claimed diff --git a/docs/guides/PARALLEL_TASKS.md b/docs/guides/PARALLEL_TASKS.md new file mode 100644 index 0000000..27453f4 --- /dev/null +++ b/docs/guides/PARALLEL_TASKS.md @@ -0,0 +1,86 @@ +# Deterministic parallel tasks + +Set `spec.runtime.maxConcurrency` between `1` and `64`. The default remains +`1`, so existing workflows keep sequential behavior. A value greater than one +allows independent ready tasks to execute in bounded batches. + +```yaml +spec: + runtime: + maxConcurrency: 2 + actions: + remember: + kind: builtin.memory.write + tasks: + - id: left + uses: action:remember + with: { key: left, value: one } + - id: right + uses: action:remember + with: { key: right, value: two } +``` + +## Deterministic boundary + +The compiler fixes task order with the declaration-order topological plan. The +runtime selects ready tasks in that order up to the concurrency limit. Every +task in a batch reads an immutable working-memory snapshot. That snapshot is +encrypted when state encryption is enabled and retained while a task is +running or waiting for approval, so resume does not substitute newer sibling +state after a crash. + +Task bodies may finish in any order. Successful outputs, state deltas, artifact +references, failures, retry states, audit events, the final working-memory +value, and a checkpoint commit in plan order in one SQLite transaction. A +failed transaction exposes none of the batch results. + +Effects keep their existing stable identity of run, task, attempt, ordinal, +operation, and input digest. Parallel execution does not combine provider +sessions or tool-call histories. Recorded replay dispatches no effects. + +## Working-memory writes + +A literal `builtin.memory.write` key is inferred into the compiled task's +`memoryWrites` set. A templated key must declare every possible key: + +```yaml +- id: selected-write + uses: action:remember + memoryWrites: [left, right] + with: + key: "${{ inputs.selected }}" + value: kept +``` + +For `maxConcurrency` greater than one, unordered tasks with overlapping write +sets fail compilation. Add a `needs` edge or use disjoint keys. There is no +implicit last-writer-wins rule and no merge strategy in this format. The +runtime also verifies the rendered key before creating an effect and verifies +the completed delta before commit. + +The write set covers run working memory only. Tasks that can touch the same +file, remote object, process resource, or other external target must be ordered +with `needs` unless that external system provides the required concurrency +control. + +## Failures, approvals, and cancellation + +A stop-on-failure task does not abandon already launched siblings. The runtime +waits for their bounded execution, commits successful sibling results with the +failure, cancels remaining tasks and pending approvals, and marks the run +failed atomically. With `failure: continue`, independent branches continue and +descendants of the failed task skip normally. + +Approval-gated tasks may pause beside completed siblings. Their immutable +execution snapshot and effect request remain durable. Resume continues only +approved tasks and never repeats a confirmed effect. Cancellation propagates +through the shared cancellation token and durably cancels every non-terminal +task. + +Failed-only retry and selective repair use the ordinary graph closure. Proven +successful parallel siblings are reusable; selected failed or changed branches +receive fresh attempts. Offline replay applies recorded state deltas in +compiled order. + +See [`examples/v1/parallel.yaml`](../../examples/v1/parallel.yaml) for a +credential-free executable example. diff --git a/docs/memory.md b/docs/memory.md index 205c1c7..e32082d 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -3,7 +3,7 @@ Four mechanisms remain intentionally separate: - Run state is authoritative lifecycle data: inputs, task states, attempts, outputs, cancellations, effects, approvals, and checkpoints. -- Working memory is a JSON object owned by one run. Writes are explicit keyed internal-state effects and the updated object commits with the task transition and checkpoint. Sequential scheduling is its merge rule. +- Working memory is a JSON object owned by one run. Writes are explicit keyed internal-state effects. Parallel tasks read durable isolated snapshots; disjoint deltas commit in compiled order with task transitions and the checkpoint. Unordered conflicting write sets fail compilation. - Long-term memory is namespaced SQLite data across runs with optional expiry. Reads/writes are explicit actions; `memory get/put` and `gc` provide administration. Replay never rolls it back or treats it as history. - Provider prompt cache is an optional performance optimization. Cache keys/options and usage counts are provider metadata, never correctness or memory. diff --git a/docs/reference/DATABASE.md b/docs/reference/DATABASE.md index 9d38c5c..ac3d856 100644 --- a/docs/reference/DATABASE.md +++ b/docs/reference/DATABASE.md @@ -1,11 +1,11 @@ # Runtime database and migrations -The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `10`. +The local SQLite database and its sibling artifact root are history and part of the correctness boundary. The current database schema version is `11`. ## Stored records - runs, source workflow, compiled plan, inputs, output, mode, state, parent linkage, and repair/retry source/root metadata -- task states, attempts, output, errors, disposition, source attempt, versioned fingerprints/digests, state delta, artifact manifest, and reuse decision +- task states, attempts, output, errors, disposition, source attempt, versioned fingerprints/digests, state delta, artifact manifest, reuse decision, and encrypted-capable execution memory snapshot - effects, request/result/error, confirmation, and uncertainty - immutable effect reconciliation history, operator authorization, evidence, validated results, supersession, and compensation linkage - approvals and resolutions @@ -18,7 +18,7 @@ The local SQLite database and its sibling artifact root are history and part of Working memory is stored on the run and in checkpoints. Provider credentials and state-encryption key values are not stored. Prompts, workflow inputs and outputs, task output and errors, effect input/results, approvals, checkpoints, audit/trace payloads, provider continuations, reconciliation evidence, and long-term-memory values can be protected with application-level authenticated envelopes. -Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. Migration 9 adds retry roots/reason/version and failed-only selection. Migration 10 adds state-encryption configuration and fail-closed write guards. A repair or retry transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete the derived run. +Migration 5 adds `source_run_id`, `source_workflow_digest`, repair roots/reason/version, and task-boundary metadata used by repair. Migration 6 adds artifact blob, reference, and ingestion-lease tables. Migration 7 records transactional legacy-run upgrades. Migration 8 adds immutable effect reconciliation records. Migration 9 adds retry roots/reason/version and failed-only selection. Migration 10 adds state-encryption configuration and fail-closed write guards. Migration 11 adds the protected execution-memory snapshot used to preserve parallel task boundaries across crashes and approvals. A repair or retry transaction creates the run, materializes every reused task and artifact reference, creates pending fresh tasks, records provenance audit events, and writes its first checkpoint atomically. The source identifier is durable lineage rather than a foreign-key dependency, so source garbage collection does not delete the derived run. Artifact manifests contain logical path/name, media type, byte size, SHA-256 digest, and CAS-relative path. Blob bytes live under `/artifacts/sha256/`; identical content is stored once. A completed repair/replay receives its own references, so source-row and workspace deletion do not break it. diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index b6b0298..4fa87db 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -31,7 +31,7 @@ Unknown fields fail. Documents, ordinary input files, packs, direct reads, exist | `mcpServers` | `{}` | Pinned MCP Streamable HTTP peers. | | `a2aPeers` | `{}` | Pinned A2A Agent Card peers. | | `packs` | `[]` | Local reviewed pack references. | -| `runtime` | sequential defaults | Runtime controls. `maxConcurrency` must be `1`. | +| `runtime` | bounded defaults | Runtime controls. `maxConcurrency` defaults to `1` and accepts `1` through `64`. | | `output` | defaults | Output presentation contract. | ## Tasks @@ -41,6 +41,7 @@ Each task requires `id` and `uses`. `uses` is `action:name` or `agent:name`. | Field | Default | Validation | | --- | --- | --- | | `needs` | `[]` | Every ID must exist; cycles fail. | +| `memoryWrites` | inferred or `[]` | Working-memory keys. Literal memory-write keys are inferred; templated keys require an explicit set. Unordered overlaps fail when concurrency is greater than one. | | `when` | true | Constrained boolean/equality expression. | | `vars` | `{}` | Task-local JSON values. | | `with` | `{}` | Typed action or agent input. | @@ -49,7 +50,10 @@ Each task requires `id` and `uses`. `uses` is `action:name` or `agent:name`. | `timeoutSeconds` | action or agent default | Must be within the implementation bound. | | failure behavior | fail | Unsupported dynamic control flow is rejected. | -Ready tasks run in YAML declaration order. There is no `foreach`, matrix, loop, router, sub-workflow, handler, or parallel group in this version. +Ready tasks are selected in YAML declaration order up to `maxConcurrency`. +They read isolated durable snapshots and commit in compiled order. There is no +`foreach`, matrix, loop, router, sub-workflow, handler, or separate parallel +group in this version. ## Agents diff --git a/docs/research/LANDSCAPE.md b/docs/research/LANDSCAPE.md index 020ec99..bcc0b20 100644 --- a/docs/research/LANDSCAPE.md +++ b/docs/research/LANDSCAPE.md @@ -5,7 +5,7 @@ Research was performed against official documentation and stable specifications | Pattern | Source | Problem solved | Existing agentctl equivalent | Decision | Product rationale | Technical consequence | Compatibility impact | Security impact | Testing implication | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | Typed workflows separate from agents | [Agno workflows](https://docs.agno.com/workflows2/overview) | Keeps orchestration outside model reasoning | Workflow/task/agent types | Adapt | The graph must remain authoritative | Compiler schedules bounded agent nodes | Clarifies legacy agent steps | Removes model control of scheduler | Compiler and bounded-loop tests | -| Sequential, conditional, parallel steps | [Agno workflow patterns](https://docs.agno.com/workflows2) | Express common control flow | Dependencies and `when` | Adapt; defer parallel | Sequential DAG and safe conditions suffice until merge rules exist | Reject `maxConcurrency > 1` | Legacy sequential behavior preserved | Avoids race-based policy/state bugs | Stable-order and condition tests | +| Sequential, conditional, parallel steps | [Agno workflow patterns](https://docs.agno.com/workflows2) | Express common control flow | Dependencies and `when` | Adapt | Graph parallelism stays explicit and bounded | Stable ready batches, isolated snapshots, declared memory writes, atomic plan-order commit | Default concurrency one preserves legacy behavior | Conflicts fail before effects and external sharing requires dependencies | Real overlap, conflict, crash, approval, repair, retry, and replay tests | | Idempotent modules and check/diff | [Ansible playbooks](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_intro.html) | Predict changes before mutation | Action results and check mode | Adapt | Honest changed/unchanged is useful developer feedback | Typed action result and predictability | Makes old dry-run semantics explicit | Preview cannot authorize an effect | No-mutation and diff tests | | Fully qualified reusable content | [Ansible collections](https://docs.ansible.com/ansible/latest/collections_guide/index.html) | Avoid name collision and package automation | Pack dotted names | Adopt | Local packs need stable identity | Manifest validation and integrity digest | Old slash pack refs need migration | Enables provenance policy | Tamper and semver tests | | Plan before apply | [Terraform plan](https://developer.hashicorp.com/terraform/cli/commands/plan) | Separate validation/prediction from mutation | `check`, `plan`, `run --check` | Adapt | Remote/model effects are not predictable | Plan carries predictability class | Replaces optimistic legacy preview | Prevents false safety claims | Predictability golden tests | diff --git a/examples/v1/README.md b/examples/v1/README.md index 7f41526..79d1c33 100644 --- a/examples/v1/README.md +++ b/examples/v1/README.md @@ -9,6 +9,7 @@ The deterministic examples are exercised by `cargo xtask verify` and never requi - `approval.yaml`: durable approval-gated workspace mutation. - `policy-denial.yaml`: an explicit tool-policy denial. - `crash-resume.yaml`: effect-ledger write followed by observation; crash behavior is injected in runtime tests. +- `parallel.yaml`: bounded parallel batches with disjoint working-memory writes and stable commits. - `working-memory.yaml` and `long-term-memory.yaml`: separate memory lifecycles. - `fake-provider.yaml`: deterministic model-provider path. - `mcp.yaml` and `a2a.yaml`: local protocol fixtures, backed by the protocol crate's mock-server tests. diff --git a/examples/v1/parallel.yaml b/examples/v1/parallel.yaml new file mode 100644 index 0000000..0b598f2 --- /dev/null +++ b/examples/v1/parallel.yaml @@ -0,0 +1,32 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: deterministic-parallel +spec: + runtime: + maxConcurrency: 2 + policy: + approval: never + memory: + working: + seed: durable + actions: + remember: + kind: builtin.memory.write + recall: + kind: builtin.memory.read + tasks: + - id: write-left + uses: action:remember + with: { key: left, value: one } + - id: write-right + uses: action:remember + with: { key: right, value: two } + - id: read-left + uses: action:recall + needs: [write-left, write-right] + with: { key: left } + - id: read-right + uses: action:recall + needs: [write-left, write-right] + with: { key: right } diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 7bbb570..e8557ab 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -107,6 +107,7 @@ dependencies = [ "agentctl-store", "async-trait", "chrono", + "futures-util", "hex", "jsonschema", "nix", diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index 6925fc7..61976e8 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -702,6 +702,12 @@ }, "default": [] }, + "memoryWrites": { + "type": "array", + "items": { + "type": "string" + } + }, "when": { "type": [ "string", diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 6d30062..f6e847d 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 32; +const ACCEPTANCE_SCENARIOS: usize = 33; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1312,6 +1312,60 @@ pub fn run(root: &Path) -> Result<()> { ); ensure!(!serde_json::to_string(&missing_secret)?.contains(secret_marker)); + scenario( + 33, + "packaged CLI runs, inspects, and replays an atomic parallel batch", + ); + let parallel = root.join("examples/v1/parallel.yaml"); + let parallel_plan = successful_json( + &binary, + root, + &strings([ + "plan", + path(¶llel)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(¶llel_plan, "/data/maxConcurrency", 2_u64)?; + ensure_eq( + ¶llel_plan, + "/data/tasks/write-left/memoryWrites/0", + "left", + )?; + ensure_eq( + ¶llel_plan, + "/data/tasks/write-right/memoryWrites/0", + "right", + )?; + let parallel_db = directory.path().join("parallel.db"); + let parallel_run = + successful_json(&binary, root, &run_args(¶llel, ¶llel_db, root, &[]))?; + ensure_eq(¶llel_run, "/data/state", "succeeded")?; + let parallel_run_id = string_at(¶llel_run, "/data/runId")?; + let parallel_inspect = inspect(&binary, root, ¶llel_db, parallel_run_id)?; + ensure_eq(¶llel_inspect, "/data/run/workingMemory/left", "one")?; + ensure_eq(¶llel_inspect, "/data/run/workingMemory/right", "two")?; + let parallel_replay = successful_json( + &binary, + root, + &strings([ + "replay", + parallel_run_id, + "--db", + path(¶llel_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let parallel_replay_id = string_at(¶llel_replay, "/data/runId")?; + let parallel_replay_inspect = inspect(&binary, root, ¶llel_db, parallel_replay_id)?; + ensure!(array_len(¶llel_replay_inspect, "/data/effects")? == 0); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } @@ -1340,6 +1394,38 @@ pub fn container(root: &Path) -> Result<()> { ensure!(array_len(&replay_inspect, "/data/effects")? == 0); ensure!(array_len(&replay_inspect, "/data/toolCalls")? == 0); + write( + &layout.config.join("parallel.yaml"), + &fs::read_to_string(root.join("examples/v1/parallel.yaml"))?, + )?; + let parallel = container_agentctl( + &engine, + &layout, + &[ + "run", + "/config/parallel.yaml", + "--workspace", + "/workspace", + "--db", + "/state/runtime.db", + "--output", + "json", + "--color", + "never", + ], + 0, + "OCI parallel run", + )?; + ensure_eq(¶llel, "/data/state", "succeeded")?; + let parallel_id = string_at(¶llel, "/data/runId")?; + let parallel_inspect = inspect_container(&engine, &layout, parallel_id)?; + ensure_eq(¶llel_inspect, "/data/run/workingMemory/left", "one")?; + ensure_eq(¶llel_inspect, "/data/run/workingMemory/right", "two")?; + let parallel_replay = replay_container(&engine, &layout, parallel_id)?; + let parallel_replay_id = string_at(¶llel_replay, "/data/runId")?; + let parallel_replay_inspect = inspect_container(&engine, &layout, parallel_replay_id)?; + ensure!(array_len(¶llel_replay_inspect, "/data/effects")? == 0); + let repair_directory = tempfile::tempdir()?; let repair_layout = container_layout(repair_directory.path(), false)?; write( @@ -1432,7 +1518,7 @@ pub fn container(root: &Path) -> Result<()> { container_signal_acceptance(&engine, directory.path())?; println!( - "agentctl OCI acceptance passed: success, artifact, inspect, network-disabled replay, selective repair, missing-secret, invalid-input, SIGTERM, non-root, read-only root, mounted state/artifacts" + "agentctl OCI acceptance passed: success, artifact, inspect, parallel ordered commit, network-disabled replay, selective repair, missing-secret, invalid-input, SIGTERM, non-root, read-only root, mounted state/artifacts" ); Ok(()) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e603329..4540340 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -400,6 +400,7 @@ fn verify_examples(root: &Path) -> Result<()> { "hello.yaml", "dataflow.yaml", "condition.yaml", + "parallel.yaml", "working-memory.yaml", "long-term-memory.yaml", "fake-provider.yaml", From c1a869c9f009b5b59fa0871032c5d9d0f3bef276 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 21:19:14 +0530 Subject: [PATCH 12/44] feat: add bounded task expansion --- README.md | 1 + crates/agentctl-core/src/compiler.rs | 446 ++++++++++++++++-- crates/agentctl-core/src/dsl.rs | 72 +++ crates/agentctl-core/tests/compatibility.rs | 2 +- crates/agentctl-runtime/src/lib.rs | 212 ++++++++- docs/ARCHITECTURE.md | 11 +- docs/COMPATIBILITY.md | 4 +- docs/DSL.md | 9 +- docs/DURABLE_EXECUTION.md | 6 + docs/LIMITATIONS.md | 5 +- .../adr/0009-bounded-static-task-expansion.md | 22 + docs/execution/COMPLETENESS_VERIFICATION.md | 3 +- docs/execution/DECISIONS.md | 1 + docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 16 +- docs/guides/MATRIX_AND_FOREACH.md | 83 ++++ docs/reference/YAML.md | 10 +- examples/v1/README.md | 1 + examples/v1/matrix.yaml | 26 + schemas/workflow.schema.json | 65 +++ xtask/src/acceptance.rs | 67 ++- 21 files changed, 998 insertions(+), 65 deletions(-) create mode 100644 docs/adr/0009-bounded-static-task-expansion.md create mode 100644 docs/guides/MATRIX_AND_FOREACH.md create mode 100644 examples/v1/matrix.yaml diff --git a/README.md b/README.md index f136926..1f5d9fc 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_R For confidential workflow history, use [Sensitive-state encryption](docs/guides/SENSITIVE_STATE_ENCRYPTION.md). For environment, mounted-file, and policy-gated process credentials, use [Secret references](docs/guides/SECRET_REFERENCES.md). For bounded independent branches and working-memory conflict rules, use [Deterministic parallel tasks](docs/guides/PARALLEL_TASKS.md). +For bounded static task expansion and child-level recovery, use [Matrix and foreach tasks](docs/guides/MATRIX_AND_FOREACH.md). ## Safety boundary diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index dbb58b8..ac63e36 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -7,8 +7,8 @@ use sha2::{Digest, Sha256}; use crate::PLAN_FORMAT_VERSION; use crate::diagnostic::{Diagnostic, DiagnosticCode}; use crate::dsl::{ - ActionKind, EffectClass, Idempotency, JsonMap, ProviderKind, RetryDefinition, ToolKind, - Workflow, + ActionKind, EffectClass, Idempotency, JsonMap, MAX_EXPANSION_ITEMS, ProviderKind, + RetryDefinition, TaskDefinition, ToolKind, Workflow, }; use crate::template::{TemplateError, referenced_tasks, validate_expression}; @@ -26,6 +26,8 @@ pub struct CompiledTask { pub id: String, pub uses: TaskUse, pub needs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expansion: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub memory_writes: Vec, pub when: Option, @@ -38,11 +40,20 @@ pub struct CompiledTask { pub predictability: PlanPredictability, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledExpansion { + pub parent: String, + pub index: usize, + pub bindings: JsonMap, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "kind", content = "name")] pub enum TaskUse { Action(String), Agent(String), + Aggregate(Vec), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -135,12 +146,251 @@ impl ProviderCapability { } } +#[derive(Debug, Clone)] +struct ExpandedTaskDefinition { + definition: TaskDefinition, + source_position: usize, + expansion: Option, + aggregate_children: Option>, +} + +fn expand_tasks( + workflow: &Workflow, + file: &str, + diagnostics: &mut Vec, +) -> Vec { + let mut expanded = Vec::new(); + for (position, task) in workflow.spec.tasks.iter().enumerate() { + let bindings = match (&task.foreach, &task.matrix) { + (None, None) => { + expanded.push(ExpandedTaskDefinition { + definition: task.clone(), + source_position: position, + expansion: None, + aggregate_children: None, + }); + continue; + } + (Some(_), Some(_)) => { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "task `{}` cannot declare both foreach and matrix expansion", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}]")), + ); + continue; + } + (Some(foreach), None) => { + if !valid_identifier(&foreach.binding) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("invalid foreach binding `{}`", foreach.binding), + ) + .with_path(format!("spec.tasks[{position}].foreach.as")), + ); + continue; + } + if task.vars.contains_key(&foreach.binding) + || task.vars.contains_key("foreachIndex") + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "task `{}` foreach bindings conflict with existing task vars", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}].vars")), + ); + continue; + } + if foreach.max_items == 0 + || foreach.max_items > MAX_EXPANSION_ITEMS + || foreach.items.len() > foreach.max_items + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "task `{}` foreach expands to {} items, exceeding maxItems {} or framework maximum {MAX_EXPANSION_ITEMS}", + task.id, + foreach.items.len(), + foreach.max_items + ), + ) + .with_path(format!("spec.tasks[{position}].foreach")), + ); + continue; + } + foreach + .items + .iter() + .enumerate() + .map(|(index, item)| { + let mut values = JsonMap::new(); + values.insert(foreach.binding.clone(), item.clone()); + values.insert("foreachIndex".to_owned(), Value::from(index)); + values + }) + .collect::>() + } + (None, Some(matrix)) => { + if matrix.axes.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("task `{}` matrix must declare at least one axis", task.id), + ) + .with_path(format!("spec.tasks[{position}].matrix.axes")), + ); + continue; + } + if task.vars.contains_key("matrix") || task.vars.contains_key("matrixIndex") { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "task `{}` matrix bindings conflict with existing task vars", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}].vars")), + ); + continue; + } + if let Some(axis) = matrix.axes.keys().find(|name| !valid_identifier(name)) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("invalid matrix axis `{axis}`"), + ) + .with_path(format!("spec.tasks[{position}].matrix.axes")), + ); + continue; + } + let count = matrix + .axes + .values() + .try_fold(1_usize, |count, values| count.checked_mul(values.len())); + if matrix.max_items == 0 + || matrix.max_items > MAX_EXPANSION_ITEMS + || count.is_none_or(|count| count > matrix.max_items) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "task `{}` matrix exceeds maxItems {} or framework maximum {MAX_EXPANSION_ITEMS}", + task.id, matrix.max_items + ), + ) + .with_path(format!("spec.tasks[{position}].matrix")), + ); + continue; + } + let mut combinations = vec![JsonMap::new()]; + for (axis, values) in &matrix.axes { + let mut next = Vec::with_capacity(combinations.len() * values.len()); + for combination in &combinations { + for value in values { + let mut candidate = combination.clone(); + candidate.insert(axis.clone(), value.clone()); + next.push(candidate); + } + } + combinations = next; + } + combinations + .into_iter() + .enumerate() + .map(|(index, matrix)| { + let mut values = JsonMap::new(); + values.insert( + "matrix".to_owned(), + Value::Object(matrix.into_iter().collect()), + ); + values.insert("matrixIndex".to_owned(), Value::from(index)); + values + }) + .collect::>() + } + }; + + let mut children = Vec::with_capacity(bindings.len()); + for (index, values) in bindings.into_iter().enumerate() { + let id = expanded_task_id(&task.id, index, &values); + let mut child = task.clone(); + child.id.clone_from(&id); + child.foreach = None; + child.matrix = None; + child.vars.extend(values.clone()); + children.push(id.clone()); + expanded.push(ExpandedTaskDefinition { + definition: child, + source_position: position, + expansion: Some(CompiledExpansion { + parent: task.id.clone(), + index, + bindings: values, + }), + aggregate_children: None, + }); + } + + let mut aggregate = task.clone(); + aggregate.needs.clone_from(&children); + aggregate.foreach = None; + aggregate.matrix = None; + aggregate.memory_writes.clear(); + aggregate.when = None; + aggregate.vars.clear(); + aggregate.input.clear(); + aggregate.retry = RetryDefinition::default(); + aggregate.timeout_seconds = None; + aggregate.output_schema = None; + expanded.push(ExpandedTaskDefinition { + definition: aggregate, + source_position: position, + expansion: None, + aggregate_children: Some(children), + }); + } + expanded +} + +fn expanded_task_id(parent: &str, index: usize, bindings: &JsonMap) -> String { + let encoded = serde_json::to_vec(bindings).unwrap_or_default(); + let digest = sha256(&encoded); + format!("{parent}--{index:04}-{}", &digest[..12]) +} + pub fn compile(workflow: &Workflow, file: &str) -> Result> { let mut diagnostics = Vec::new(); let mut tasks = BTreeMap::new(); let mut declaration_order = Vec::new(); + let mut source_positions = BTreeMap::new(); + let expanded_tasks = expand_tasks(workflow, file, &mut diagnostics); + if !diagnostics.is_empty() { + return Err(diagnostics); + } - for (position, task) in workflow.spec.tasks.iter().enumerate() { + for expanded in &expanded_tasks { + let position = expanded.source_position; + let task = &expanded.definition; if tasks.contains_key(&task.id) { diagnostics.push( Diagnostic::error( @@ -162,23 +412,27 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result { - TaskUse::Action(name) - } - Some(TaskUse::Agent(name)) if workflow.spec.agents.contains_key(&name) => { - TaskUse::Agent(name) - } - _ => { - diagnostics.push( - Diagnostic::error( - DiagnosticCode::MissingReference, - file, - format!("task `{}` refers to unknown `{}`", task.id, task.uses), - ) - .with_path(format!("spec.tasks[{position}].uses")), - ); - continue; + let task_use = if let Some(children) = &expanded.aggregate_children { + TaskUse::Aggregate(children.clone()) + } else { + match parse_use(&task.uses) { + Some(TaskUse::Action(name)) if workflow.spec.actions.contains_key(&name) => { + TaskUse::Action(name) + } + Some(TaskUse::Agent(name)) if workflow.spec.agents.contains_key(&name) => { + TaskUse::Agent(name) + } + _ => { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::MissingReference, + file, + format!("task `{}` refers to unknown `{}`", task.id, task.uses), + ) + .with_path(format!("spec.tasks[{position}].uses")), + ); + continue; + } } }; let mut input = match &task_use { @@ -188,18 +442,22 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + TaskUse::Agent(_) | TaskUse::Aggregate(_) => JsonMap::new(), }; input.extend(task.input.clone()); - let memory_writes = task_memory_writes( - workflow, - &task_use, - &input, - &task.memory_writes, - file, - position, - &mut diagnostics, - ); + let memory_writes = if matches!(task_use, TaskUse::Aggregate(_)) { + Vec::new() + } else { + task_memory_writes( + workflow, + &task_use, + &input, + &task.memory_writes, + file, + position, + &mut diagnostics, + ) + }; let mut vars = match &task_use { TaskUse::Agent(name) => workflow .spec @@ -207,16 +465,18 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + TaskUse::Action(_) | TaskUse::Aggregate(_) => JsonMap::new(), }; vars.extend(task.vars.clone()); declaration_order.push(task.id.clone()); + source_positions.insert(task.id.clone(), position); tasks.insert( task.id.clone(), CompiledTask { id: task.id.clone(), uses: task_use, needs: task.needs.clone(), + expansion: expanded.expansion.clone(), memory_writes, when: task.when.clone(), vars, @@ -232,10 +492,11 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result PlanPredictability::FullyPredictable, }; } let predictability = tasks @@ -385,7 +647,7 @@ fn task_memory_writes( .actions .get(name) .is_some_and(|action| action.kind == ActionKind::MemoryWrite), - TaskUse::Agent(_) => false, + TaskUse::Agent(_) | TaskUse::Aggregate(_) => false, }; if !is_memory_write { if !declared.is_empty() { @@ -591,6 +853,7 @@ fn plan_requirements( predictability: task.predictability, }] } + TaskUse::Aggregate(_) => Vec::new(), }) .collect(); PlanRequirements { @@ -1220,6 +1483,123 @@ spec: assert_eq!(plan.order, ["b", "a", "c"]); } + #[test] + fn foreach_and_matrix_expand_to_stable_children_and_aggregates() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: expansion } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: each + uses: "action:assign" + foreach: + items: [alpha, beta] + as: value + maxItems: 2 + with: { value: "${{ vars.value }}", index: "${{ vars.foreachIndex }}" } + - id: combinations + uses: "action:assign" + matrix: + axes: + os: [linux, macos] + tier: [small, large] + maxItems: 4 + with: + os: "${{ vars.matrix.os }}" + tier: "${{ vars.matrix.tier }}" + index: "${{ vars.matrixIndex }}" + - { id: done, uses: "action:assign", needs: [each, combinations] } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("compiles"); + let second = compile(&workflow, "fixture.yaml").expect("compiles deterministically"); + assert_eq!(plan, second); + + let each_children = match &plan.tasks["each"].uses { + TaskUse::Aggregate(children) => children, + other => panic!("expected foreach aggregate, got {other:?}"), + }; + assert_eq!(each_children.len(), 2); + assert!(each_children[0].starts_with("each--0000-")); + assert_eq!( + plan.tasks[&each_children[1]] + .expansion + .as_ref() + .expect("expansion") + .bindings["value"], + Value::String("beta".to_owned()) + ); + assert_eq!(plan.tasks["each"].needs, *each_children); + + let matrix_children = match &plan.tasks["combinations"].uses { + TaskUse::Aggregate(children) => children, + other => panic!("expected matrix aggregate, got {other:?}"), + }; + assert_eq!(matrix_children.len(), 4); + assert_eq!( + plan.tasks[&matrix_children[2]] + .expansion + .as_ref() + .expect("expansion") + .bindings["matrix"], + serde_json::json!({"os": "macos", "tier": "small"}) + ); + assert_eq!( + plan.tasks["done"].needs, + ["each".to_owned(), "combinations".to_owned()] + ); + } + + #[test] + fn expansion_bounds_and_binding_collisions_are_rejected() { + let too_many = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: expansion-bound } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: each + uses: "action:assign" + foreach: { items: [a, b], maxItems: 1 } +"#, + ); + let diagnostics = compile(&too_many, "fixture.yaml").expect_err("bound rejected"); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("foreach expands to 2 items")) + ); + + let collision = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: expansion-collision } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: each + uses: "action:assign" + vars: { item: existing } + foreach: { items: [a] } +"#, + ); + let diagnostics = compile(&collision, "fixture.yaml").expect_err("collision rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("foreach bindings conflict with existing task vars") + })); + } + #[test] fn parallel_memory_writes_are_inferred_and_conflicts_are_rejected() { let workflow = parse( diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index f438101..37f6764 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -420,6 +420,10 @@ pub struct TaskDefinition { pub uses: String, #[serde(default)] pub needs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub foreach: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matrix: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub memory_writes: Vec, #[serde(default)] @@ -438,6 +442,35 @@ pub struct TaskDefinition { pub output_schema: Option, } +pub const DEFAULT_MAX_EXPANSION_ITEMS: usize = 32; +pub const MAX_EXPANSION_ITEMS: usize = 256; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ForeachDefinition { + pub items: Vec, + #[serde(default = "default_item_binding", rename = "as")] + pub binding: String, + #[serde(default = "default_max_expansion_items")] + pub max_items: usize, +} + +fn default_item_binding() -> String { + "item".to_owned() +} + +const fn default_max_expansion_items() -> usize { + DEFAULT_MAX_EXPANSION_ITEMS +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MatrixDefinition { + pub axes: BTreeMap>, + #[serde(default = "default_max_expansion_items")] + pub max_items: usize, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RetryDefinition { @@ -916,6 +949,45 @@ fn validate_document(workflow: &Workflow, file: &str) -> Vec { .with_path("spec.runtime.defaultTimeoutSeconds"), ); } + for (position, task) in workflow.spec.tasks.iter().enumerate() { + if task.foreach.is_some() && task.matrix.is_some() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "task `{}` cannot declare both foreach and matrix expansion", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}]")), + ); + } + if let Some(foreach) = &task.foreach + && (foreach.max_items == 0 || foreach.max_items > MAX_EXPANSION_ITEMS) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("foreach.maxItems must be between 1 and {MAX_EXPANSION_ITEMS}"), + ) + .with_path(format!("spec.tasks[{position}].foreach.maxItems")), + ); + } + if let Some(matrix) = &task.matrix + && (matrix.max_items == 0 || matrix.max_items > MAX_EXPANSION_ITEMS) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("matrix.maxItems must be between 1 and {MAX_EXPANSION_ITEMS}"), + ) + .with_path(format!("spec.tasks[{position}].matrix.maxItems")), + ); + } + } for (name, action) in &workflow.spec.actions { if let Err(message) = action.validate_process_bounds() { diagnostics.push( diff --git a/crates/agentctl-core/tests/compatibility.rs b/crates/agentctl-core/tests/compatibility.rs index dfd2319..b1e3bbe 100644 --- a/crates/agentctl-core/tests/compatibility.rs +++ b/crates/agentctl-core/tests/compatibility.rs @@ -39,6 +39,6 @@ fn typescript_assign_fixture_translates_to_the_language_neutral_contract() { assert_eq!(expected.task.use_kind, "action"); assert_eq!(reference, &expected.task.reference); } - TaskUse::Agent(_) => panic!("expected action task"), + TaskUse::Agent(_) | TaskUse::Aggregate(_) => panic!("expected action task"), } } diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index d24790f..b1d3be9 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -2461,12 +2461,14 @@ impl Runtime { { continue; } - if dependencies.iter().any(|dependency| { - matches!( - dependency.state, - TaskState::Failed | TaskState::Cancelled | TaskState::Skipped - ) - }) { + if !matches!(task.uses, TaskUse::Aggregate(_)) + && dependencies.iter().any(|dependency| { + matches!( + dependency.state, + TaskState::Failed | TaskState::Cancelled | TaskState::Skipped + ) + }) + { self.store.transition_task( run_id, task_id, @@ -2976,12 +2978,14 @@ impl Runtime { .iter() .filter_map(|needed| tasks.iter().find(|candidate| &candidate.task_id == needed)) .collect(); - if dependencies.iter().any(|dependency| { - matches!( - dependency.state, - TaskState::Failed | TaskState::Cancelled | TaskState::Skipped - ) - }) { + if !matches!(task.uses, TaskUse::Aggregate(_)) + && dependencies.iter().any(|dependency| { + matches!( + dependency.state, + TaskState::Failed | TaskState::Cancelled | TaskState::Skipped + ) + }) + { self.store.transition_task( run_id, &task.id, @@ -3371,6 +3375,34 @@ impl Runtime { ) .await } + TaskUse::Aggregate(children) => { + let records = self.store.list_tasks(&run.run_id)?; + let items = children + .iter() + .enumerate() + .map(|(index, child)| { + let record = records + .iter() + .find(|candidate| &candidate.task_id == child) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "expanded child task `{child}` disappeared" + )) + })?; + Ok(serde_json::json!({ + "index": index, + "taskId": child, + "state": record.state, + "output": record.output, + "error": record.error, + })) + }) + .collect::, RuntimeError>>()?; + Ok(TaskExecution::Complete { + output: serde_json::json!({"items": items}), + memory: None, + }) + } } } @@ -4836,6 +4868,12 @@ fn task_output_schema(workflow: &Workflow, task: &agentctl_core::CompiledTask) - .get(name) .and_then(|agent| agent.structured_output.clone()), TaskUse::Action(_) => Some(serde_json::json!({"type": "object"})), + TaskUse::Aggregate(_) => Some(serde_json::json!({ + "type": "object", + "required": ["items"], + "properties": {"items": {"type": "array"}}, + "additionalProperties": false + })), }) } @@ -4920,6 +4958,11 @@ fn task_definition_fingerprint( "instructionContentDigest": instruction_content_digest, }) } + TaskUse::Aggregate(children) => serde_json::json!({ + "kind": "aggregate", + "task": task, + "children": children, + }), }; versioned_json_digest(&serde_json::json!({ "formatVersion": 1, @@ -8764,6 +8807,151 @@ spec: assert_eq!(replay.output, outcome.output); } + #[tokio::test] + async fn foreach_aggregates_partial_results_and_retries_only_failed_children() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(TerminalRetryProvider::default()); + let runtime = runtime(store.clone(), directory.path()) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let (workflow, compiled) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: foreach-retry } +spec: + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: return a recovery object + structuredOutput: + type: object + required: [value] + additionalProperties: false + properties: + value: { type: string } + tasks: + - id: expanded + uses: "agent:worker" + foreach: + items: [first, second] + as: item + maxItems: 2 + with: { prompt: "${{ vars.item }}" } + failure: continue +"#, + ); + let children = match &compiled.tasks["expanded"].uses { + TaskUse::Aggregate(children) => children.clone(), + other => panic!("expected aggregate, got {other:?}"), + }; + let source = runtime + .start( + &workflow, + &compiled, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("continue failure returns a terminal outcome"); + assert_eq!(source.state, RunState::Failed); + let source_tasks = store.list_tasks(&source.run_id).expect("source tasks"); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == children[0]) + .expect("first child") + .state, + TaskState::Failed + ); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == children[1]) + .expect("second child") + .state, + TaskState::Succeeded + ); + let aggregate = source_tasks + .iter() + .find(|task| task.task_id == "expanded") + .expect("aggregate"); + assert_eq!(aggregate.state, TaskState::Succeeded); + assert_eq!( + aggregate.output.as_ref().expect("aggregate output")["items"][0]["state"], + "failed" + ); + + let repair_plan = runtime + .plan_repair( + &source.run_id, + &workflow, + &compiled, + &[children[0].clone()], + false, + ) + .expect("repair plan"); + assert!(repair_plan.compatible, "{:?}", repair_plan.blocked_reuse); + assert!(repair_plan.reused_tasks.contains(&children[1])); + assert!(repair_plan.rerun_tasks.contains(&"expanded".to_owned())); + let repaired = runtime + .repair( + &workflow, + &compiled, + repair_plan, + Some("repair failed foreach child"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair succeeds"); + assert_eq!(repaired.state, RunState::Succeeded); + + let retry_plan = runtime + .plan_retry(&source.run_id, &workflow, &compiled, &[], true, false) + .expect("retry plan"); + assert_eq!(retry_plan.retry_roots, [children[0].clone()]); + assert!(retry_plan.reused_tasks.contains(&children[1])); + assert!(retry_plan.rerun_tasks.contains(&"expanded".to_owned())); + let retried = runtime + .retry( + &workflow, + &compiled, + retry_plan, + Some("retry failed foreach child"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("retry succeeds"); + assert_eq!(retried.state, RunState::Succeeded); + assert_eq!(provider.0.load(Ordering::SeqCst), 4); + let retry_tasks = store.list_tasks(&retried.run_id).expect("retry tasks"); + assert_eq!( + retry_tasks + .iter() + .find(|task| task.task_id == children[1]) + .expect("reused child") + .disposition, + TaskDisposition::Reused + ); + let replay = runtime + .replay(&retried.run_id) + .await + .expect("offline replay"); + assert_eq!(replay.state, RunState::Succeeded); + assert!( + store + .list_effects(&replay.run_id) + .expect("replay effects") + .is_empty() + ); + } + #[tokio::test] async fn retry_planning_enforces_identity_roots_acknowledgement_and_reconciliation() { let directory = tempdir().expect("tempdir"); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3d30299..906f33a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,10 +32,13 @@ Unordered overlapping `memoryWrites` fail compilation. Effects and provider sessions remain task-local. See ADR 0008 and [Deterministic parallel tasks](guides/PARALLEL_TASKS.md). -Loops, matrix/foreach expansion, routers, sub-workflows, handlers, -compensation execution, and event triggers still require their own explicit -state and recovery contracts. The DSL carries optional compensation metadata -on a tool contract, but the runtime does not execute compensation. +Static foreach lists and matrix axes compile into ordinary namespaced child +tasks followed by a pure aggregate. Their IDs, bindings, attempts, outputs, +and recovery lineage use the same durable task model as authored nodes. Loops, +routers, sub-workflows, handlers, compensation execution, and event triggers +still require their own explicit state and recovery contracts. The DSL carries +optional compensation metadata on a tool contract, but the runtime does not +execute compensation. Clock and identifier generation are injected. Provider responses, tools, and external actions are injected interfaces. Cryptographic digests canonicalize identity; output maps use stable ordering where the public contract requires it. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index ec34653..8a2ae4f 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -2,7 +2,7 @@ ## Preserved -Declaration-order scheduling among ready tasks, `needs` dataflow, exact typed templates, deterministic assign/assert/file/memory use cases, bounded agent/tool turns, approval concepts, SQLite local persistence, and the useful top-level command names remain. The language-neutral fixture records the legacy assign workflow’s translated model, graph order, and task reference. +Declaration-order scheduling among ready tasks, `needs` dataflow, exact typed templates, deterministic assign/assert/file/memory use cases, bounded agent/tool turns, approval concepts, SQLite local persistence, and the useful top-level command names remain. The language-neutral fixture records the legacy assign workflow’s translated model, graph order, and task reference. Omitted `foreach` and `matrix` fields preserve the unchanged single-task graph; compiled expansion metadata is additive. ## Migrated @@ -22,4 +22,4 @@ Legacy workflows depending on packs, broad built-in tool profiles, remote MCP/A2 ## Deferred product decisions -Foreach/matrix, loops, routers, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. +Loops, routers, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. diff --git a/docs/DSL.md b/docs/DSL.md index c24e120..59eb97c 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,7 +2,7 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, optional bounded `foreach` or `matrix` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. @@ -19,8 +19,13 @@ trace, or inspection output. See [Secret references](guides/SECRET_REFERENCES.md The compiler validates missing references, duplicate tasks, cycles, task-aware templates, tool references, provider capabilities, agent limits, concurrency bounds, and working-memory conflicts before execution. `maxConcurrency` accepts `1` through `64` and defaults to `1`. Independent ready tasks are selected in compiled order, execute against durable isolated memory snapshots, and commit atomically in compiled order. Literal working-memory keys are inferred; templated keys require `memoryWrites`. See [Deterministic parallel tasks](guides/PARALLEL_TASKS.md). +Static `foreach` lists and matrix axes expand at compile time into stable child +tasks plus a parent aggregate. `maxItems` defaults to 32, expansion cannot +exceed 256 children, and model output cannot drive it. Retry and repair can +select the visible child IDs. See [Matrix and foreach tasks](guides/MATRIX_AND_FOREACH.md). + `builtin.shell.exec` captures stdout and stderr concurrently. Its optional `stdoutLimitBytes`, `stderrLimitBytes`, and `combinedOutputLimitBytes` fields default to 1 MiB, 1 MiB, and 2 MiB respectively. Each configured value must be between 1 byte and 16 MiB. `timeoutSeconds` must be between 1 and 86,400. Exceeding an output bound terminates and reaps the process and records a structured failed effect; timeout or cancellation remains an uncertain effect because external changes may already have occurred. These fields are validated identically for workflow and pack actions. The parser translates a limited unversioned `playbook:` document and emits a migration warning. Use `agentctl migrate old.yaml --write new.yaml`. Legacy pack-backed, MCP, A2A, provider-specific, and broad module configurations need manual migration; see [Migrating from TypeScript](MIGRATING_FROM_TYPESCRIPT.md). -Not implemented in v1alpha1: `foreach`, matrix expansion, routers, loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. +Not implemented in v1alpha1: routers, loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index 9d80108..b29aac6 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -31,4 +31,10 @@ persisted immutable memory snapshot, while task output, disjoint memory deltas, artifacts, failures, audit events, and the checkpoint commit atomically in compiled order. +Static foreach and matrix declarations compile before a run is created. Every +expanded child is a normal durable task with its own attempts, effects, +fingerprint, output, retry/repair identity, and replay record. The parent is a +pure aggregate task that records child IDs, states, outputs, and errors in +stable expansion order. + The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 9f3458e..3bdc9c9 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -23,7 +23,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an These are useful extensions but are not required by the product thesis. They need new deterministic state and compatibility contracts before implementation: -- `foreach` and matrix expansion; loops; routers; sub-workflows; compensation execution; +- loops; routers; sub-workflows; compensation execution; - structured agent teams and handoffs; - model token streaming into CLI/workflow state; - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; @@ -43,6 +43,9 @@ These are useful extensions but are not required by the product thesis. They nee - The document API is `v1alpha1`; pin the binary/image version and validate before upgrading. - Parallel scheduling is local to one run and process, bounded at 64 tasks, and defaults to sequential execution. Working-memory conflicts fail compilation, but tasks that target the same external resource still require explicit `needs` ordering or that system's concurrency controls. Separate runs also require external overlap controls when effects must not overlap. +- Foreach and matrix expansion accepts only static workflow values, requires + `maxItems`, and is capped at 256 children. Runtime or model-controlled graph + growth is not supported. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. diff --git a/docs/adr/0009-bounded-static-task-expansion.md b/docs/adr/0009-bounded-static-task-expansion.md new file mode 100644 index 0000000..5f7edc2 --- /dev/null +++ b/docs/adr/0009-bounded-static-task-expansion.md @@ -0,0 +1,22 @@ +# ADR 0009: bounded static task expansion + +Status: accepted + +## Decision + +`foreach` item lists and matrix axes expand during compilation. Each child has +a stable parent, ordinal, binding digest, and ordinary durable task record. +The authored parent ID becomes a pure ordered aggregate of the child records. +Every declaration provides `maxItems`, and the framework rejects more than 256 +children. + +## Consequences + +- Model output cannot grow the graph. +- Child retry, repair, replay, effects, policy, cancellation, and inspection + reuse the existing task model. +- Downstream dependencies remain attached to the authored parent ID. +- `failure: continue` allows aggregation of partial results, but a run with a + failed child still finishes failed. +- Changing an item or axis changes the affected child identity and workflow + digest. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 9c93d58..76ecf3d 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -74,7 +74,8 @@ cargo xtask acceptance-container | Terminal retry | runtime/store identity, roots, acknowledgements, reconciliation, lineage, source immutability, and replay tests passed | packaged CLI scenario 30 and the 12-stage verification gate passed | verified | | Sensitive-state encryption | authenticated context, wrong-key, tamper, inventory, stale-writer trigger, rollback, rotation, checkpoint, and retained-schema tests passed | packaged CLI scenario 31 and the 12-stage verification gate passed | verified | | Secret references | environment compatibility, file bounds/missing/symlink containment, process allowlist/timeout/output/cancellation, zeroizing values, adapter redaction, and raw-database absence tests passed | packaged CLI scenario 32 and the 12-stage verification gate passed | verified | -| Parallel/dynamic workflows | pending | pending | open | +| Parallel scheduling | overlap, caps, conflicts, ordered atomic commits, approvals, cancellation, retry, repair, and replay tests passed | packaged CLI scenario 33 and OCI parallel run/replay passed | deterministic verified; live pending | +| Foreach/matrix | compiler bounds/identity tests and runtime partial-failure, child retry, sibling reuse, aggregation, and replay tests passed | packaged CLI scenario 34 pending | deterministic in progress | | Conditions/loops/sub-workflows | pending | pending | open | | Compensation/handoffs/streaming | pending | pending | open | | MCP/A2A resilience | pending | pending | open | diff --git a/docs/execution/DECISIONS.md b/docs/execution/DECISIONS.md index 54221a1..7092dfe 100644 --- a/docs/execution/DECISIONS.md +++ b/docs/execution/DECISIONS.md @@ -10,5 +10,6 @@ | [0006](../adr/0006-schedulable-runtime-and-noninteractive-contract.md) | Schedulable runtime, durable non-interactive pause | accepted | External platforms schedule; CLI never prompts or auto-approves in CI. | | [0007](../adr/0007-generic-oci-step-contract.md) | Generic OCI step contract | accepted | Non-root/read-only image; mounted config/workspace/state/artifacts. | | [0008](../adr/0008-deterministic-parallel-batches.md) | Deterministic parallel batches | accepted | Bounded overlap, isolated snapshots, declared writes, and atomic plan-order commits. | +| [0009](../adr/0009-bounded-static-task-expansion.md) | Bounded static task expansion | accepted | Stable child tasks and ordered aggregates prevent model-controlled graph growth. | These decisions resolve the researched patterns in [LANDSCAPE.md](../research/LANDSCAPE.md). No unsafe code or distributed control plane ADR is required because neither exists. diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md index a5d5290..fa51890 100644 --- a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -38,6 +38,7 @@ This inventory is enforced by `cargo xtask examples-verify`. The default command | `examples/v1/google-live.yaml` | Google native provider | Google | credentialed execution | 0 | 0 | N/A | Protocol mock | External opt-in | N/A | N/A | Static | passed | | `examples/v1/hello.yaml` | Minimal assign workflow | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | | `examples/v1/long-term-memory.yaml` | Namespaced durable memory | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | +| `examples/v1/matrix.yaml` | Bounded static matrix and ordered aggregation | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Aggregate states and values | passed | | `examples/v1/mcp.yaml` | MCP call contract | MCP | external execution | 0 | 0 | Protocol mock | Protocol mock | N/A | N/A | N/A | Static | passed | | `examples/v1/openai-live.yaml` | Minimal OpenAI response | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Live gate | live passed | | `examples/v1/parallel.yaml` | Deterministic parallel batch | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Atomic ordered memory merge | Canonical | passed | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index e5d84a1..55a4fcd 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -44,7 +44,7 @@ complete, every entry must have exactly one final disposition: | ISO-001 | Process isolation | open | redesigned | | BUD-001 | Resource and cost budgets | open | implemented | | SCH-001 | Deterministic parallel execution | in progress | implemented | -| DYN-001 | Foreach and matrix | open | implemented | +| DYN-001 | Foreach and matrix | in progress | implemented | | COND-001 | Conditions and routers | open | implemented | | LOOP-001 | Bounded loops | open | implemented | | SUB-001 | Sub-workflows | open | implemented | @@ -235,13 +235,14 @@ complete, every entry must have exactly one final disposition: ### DYN-001: Bounded foreach and matrix expansion -- Current behavior: no dynamic task expansion. +- Current behavior: static typed foreach lists and matrix axes compile to + bounded durable child tasks and an ordered aggregate. - User impact: authors duplicate similar tasks and cannot retry individual expanded units. - Security or durability impact: model-controlled unbounded expansion could exhaust resources. -- Product decision: compile static matrices and deterministically expand - runtime arrays only from typed non-model inputs or validated bounded outputs. +- Product decision: compile static foreach and matrix values only. Runtime or + model-controlled graph growth is outside the supported surface. - Required implementation: stable escaped child IDs, item binding, count limits, aggregate output, partial-failure rules, child inspection, repair/retry/replay, and task budgets. @@ -251,7 +252,12 @@ complete, every entry must have exactly one final disposition: - Examples: small deterministic and agent matrices. - Live evidence: two-item OpenAI matrix. - Documentation: syntax, limits, IDs, and recovery. -- Final disposition: pending implementation evidence. +- Final disposition: implemented. Static typed lists and Cartesian axes compile + to stable digest-qualified child IDs and an ordered parent aggregate. + Deterministic verification covers bounds, malformed bindings, identity, + output aggregation, partial failure, failed-only child retry, sibling reuse, + and offline replay. Program state remains in progress until the bounded live + OpenAI matrix scenario executes. ### COND-001: Typed conditions and routers diff --git a/docs/guides/MATRIX_AND_FOREACH.md b/docs/guides/MATRIX_AND_FOREACH.md new file mode 100644 index 0000000..08ccc41 --- /dev/null +++ b/docs/guides/MATRIX_AND_FOREACH.md @@ -0,0 +1,83 @@ +# Matrix and foreach tasks + +`foreach` and `matrix` expand a task into a bounded set of ordinary compiled +tasks. Expansion happens during compilation, so model output cannot create an +unbounded graph. + +## Foreach + +```yaml +tasks: + - id: inspect + uses: agent:inspector + foreach: + items: [api, worker] + as: service + maxItems: 2 + with: + prompt: "Inspect ${{ vars.service }}" +``` + +Each child receives the typed item as `vars.service` and its zero-based +position as `vars.foreachIndex`. + +## Matrix + +```yaml +tasks: + - id: verify + uses: action:assign + matrix: + axes: + platform: [linux, macos] + profile: [debug, release] + maxItems: 4 + with: + platform: "${{ vars.matrix.platform }}" + profile: "${{ vars.matrix.profile }}" + index: "${{ vars.matrixIndex }}" +``` + +Axes are traversed by axis name, then in each declared value order. `maxItems` +defaults to 32. The Cartesian product must fit that bound and the framework +maximum of 256. An empty axis produces an empty aggregate. + +## Identity and aggregation + +Compiled child IDs have the form +`PARENT--INDEX-BINDING_DIGEST`. The parent ID remains present as a pure +aggregate task. Its output is: + +```json +{ + "items": [ + { + "index": 0, + "taskId": "verify--0000-...", + "state": "succeeded", + "output": {}, + "error": null + } + ] +} +``` + +Downstream tasks continue to depend on the parent ID and read +`tasks.PARENT.output.items`. Child IDs and bindings are visible in +`agentctl plan` and `agentctl inspect`. + +## Failure and recovery + +The expanded task's `failure` setting applies to every child. With `stop`, a +failed child stops the run. With `continue`, remaining children run and the +aggregate records every child state, output, and error; the run still finishes +failed because at least one task failed. + +Retry and repair operate on child IDs. A failed-only retry reruns only failed +children and their aggregate while reusing compatible successful siblings. +Recorded replay reuses the aggregate and child outputs without provider or +tool effects. + +Expansion bindings cannot shadow task variables. `foreach` and `matrix` are +mutually exclusive on a task. Runtime expansion from model-controlled arrays +is intentionally rejected; author a static bounded list or matrix instead. diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index 4fa87db..df86e37 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -41,6 +41,8 @@ Each task requires `id` and `uses`. `uses` is `action:name` or `agent:name`. | Field | Default | Validation | | --- | --- | --- | | `needs` | `[]` | Every ID must exist; cycles fail. | +| `foreach` | none | Static typed `items`, binding `as`, and `maxItems`. Mutually exclusive with `matrix`; maximum 256 children. | +| `matrix` | none | Static `axes` Cartesian product and `maxItems`. Axis names are template-safe identifiers; maximum 256 children. | | `memoryWrites` | inferred or `[]` | Working-memory keys. Literal memory-write keys are inferred; templated keys require an explicit set. Unordered overlaps fail when concurrency is greater than one. | | `when` | true | Constrained boolean/equality expression. | | `vars` | `{}` | Task-local JSON values. | @@ -52,8 +54,9 @@ Each task requires `id` and `uses`. `uses` is `action:name` or `agent:name`. Ready tasks are selected in YAML declaration order up to `maxConcurrency`. They read isolated durable snapshots and commit in compiled order. There is no -`foreach`, matrix, loop, router, sub-workflow, handler, or separate parallel -group in this version. +runtime or model-controlled expansion. Static `foreach` and `matrix` tasks +compile to inspectable child tasks and a parent aggregate. There is no loop, +router, sub-workflow, handler, or separate parallel group in this version. ## Agents @@ -117,6 +120,7 @@ agentctl plan examples/v1/dataflow.yaml agentctl run examples/v1/dataflow.yaml --db /tmp/dataflow.db --output json --color never ``` -Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Secret +Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Matrix +and foreach](../guides/MATRIX_AND_FOREACH.md), [Secret references](../guides/SECRET_REFERENCES.md), [Policies](../policies.md), [Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). diff --git a/examples/v1/README.md b/examples/v1/README.md index 79d1c33..0611949 100644 --- a/examples/v1/README.md +++ b/examples/v1/README.md @@ -10,6 +10,7 @@ The deterministic examples are exercised by `cargo xtask verify` and never requi - `policy-denial.yaml`: an explicit tool-policy denial. - `crash-resume.yaml`: effect-ledger write followed by observation; crash behavior is injected in runtime tests. - `parallel.yaml`: bounded parallel batches with disjoint working-memory writes and stable commits. +- `matrix.yaml`: bounded static matrix expansion, stable child identities, and ordered aggregation. - `working-memory.yaml` and `long-term-memory.yaml`: separate memory lifecycles. - `fake-provider.yaml`: deterministic model-provider path. - `mcp.yaml` and `a2a.yaml`: local protocol fixtures, backed by the protocol crate's mock-server tests. diff --git a/examples/v1/matrix.yaml b/examples/v1/matrix.yaml new file mode 100644 index 0000000..294a377 --- /dev/null +++ b/examples/v1/matrix.yaml @@ -0,0 +1,26 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: bounded-matrix +spec: + runtime: + maxConcurrency: 2 + policy: + approval: never + outputs: + results: "${{ tasks.verify.output.items }}" + actions: + assign: + kind: builtin.assign + tasks: + - id: verify + uses: action:assign + matrix: + axes: + platform: [linux, macos] + profile: [debug, release] + maxItems: 4 + with: + platform: "${{ vars.matrix.platform }}" + profile: "${{ vars.matrix.profile }}" + index: "${{ vars.matrixIndex }}" diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index 61976e8..7526db7 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -702,6 +702,26 @@ }, "default": [] }, + "foreach": { + "anyOf": [ + { + "$ref": "#/$defs/ForeachDefinition" + }, + { + "type": "null" + } + ] + }, + "matrix": { + "anyOf": [ + { + "$ref": "#/$defs/MatrixDefinition" + }, + { + "type": "null" + } + ] + }, "memoryWrites": { "type": "array", "items": { @@ -753,6 +773,51 @@ "uses" ] }, + "ForeachDefinition": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": true + }, + "as": { + "type": "string", + "default": "item" + }, + "maxItems": { + "type": "integer", + "format": "uint", + "minimum": 0, + "default": 32 + } + }, + "additionalProperties": false, + "required": [ + "items" + ] + }, + "MatrixDefinition": { + "type": "object", + "properties": { + "axes": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": true + } + }, + "maxItems": { + "type": "integer", + "format": "uint", + "minimum": 0, + "default": 32 + } + }, + "additionalProperties": false, + "required": [ + "axes" + ] + }, "FailureBehavior": { "type": "string", "enum": [ diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index f6e847d..359e235 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 33; +const ACCEPTANCE_SCENARIOS: usize = 34; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1366,6 +1366,71 @@ pub fn run(root: &Path) -> Result<()> { let parallel_replay_inspect = inspect(&binary, root, ¶llel_db, parallel_replay_id)?; ensure!(array_len(¶llel_replay_inspect, "/data/effects")? == 0); + scenario( + 34, + "packaged CLI expands, aggregates, inspects, and replays a bounded matrix", + ); + let matrix = root.join("examples/v1/matrix.yaml"); + let matrix_plan = successful_json( + &binary, + root, + &strings([ + "plan", + path(&matrix)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let matrix_children = matrix_plan + .pointer("/data/tasks/verify/uses/name") + .and_then(Value::as_array) + .context("matrix aggregate child list")?; + ensure!(matrix_children.len() == 4); + let first_matrix_child = matrix_children[0] + .as_str() + .context("first matrix child ID")?; + ensure!(first_matrix_child.starts_with("verify--0000-")); + ensure_eq( + &matrix_plan, + &format!("/data/tasks/{first_matrix_child}/expansion/parent"), + "verify", + )?; + let matrix_db = directory.path().join("matrix.db"); + let matrix_run = successful_json(&binary, root, &run_args(&matrix, &matrix_db, root, &[]))?; + ensure_eq(&matrix_run, "/data/state", "succeeded")?; + ensure_eq( + &matrix_run, + "/data/output/results/0/output/output/platform", + "linux", + )?; + ensure_eq( + &matrix_run, + "/data/output/results/3/output/output/profile", + "release", + )?; + let matrix_run_id = string_at(&matrix_run, "/data/runId")?; + let matrix_inspect = inspect(&binary, root, &matrix_db, matrix_run_id)?; + ensure!(array_len(&matrix_inspect, "/data/tasks")? == 5); + let matrix_replay = successful_json( + &binary, + root, + &strings([ + "replay", + matrix_run_id, + "--db", + path(&matrix_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + let matrix_replay_id = string_at(&matrix_replay, "/data/runId")?; + let matrix_replay_inspect = inspect(&binary, root, &matrix_db, matrix_replay_id)?; + ensure!(array_len(&matrix_replay_inspect, "/data/effects")? == 0); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } From d51de8159e4f21f66c303f71a58a66f1f7b8b034 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 21:36:25 +0530 Subject: [PATCH 13/44] feat: add typed workflow routing --- README.md | 1 + crates/agentctl-core/src/compiler.rs | 333 ++++++++++++++- crates/agentctl-core/src/dsl.rs | 18 + crates/agentctl-core/tests/compatibility.rs | 4 +- crates/agentctl-runtime/src/lib.rs | 379 ++++++++++++++++-- crates/agentctl-store/src/lib.rs | 7 +- docs/ARCHITECTURE.md | 8 +- docs/COMPATIBILITY.md | 2 +- docs/DSL.md | 10 +- docs/DURABLE_EXECUTION.md | 6 + docs/LIMITATIONS.md | 5 +- ...010-typed-routing-and-durable-decisions.md | 24 ++ docs/execution/COMPLETENESS_VERIFICATION.md | 3 +- docs/execution/DECISIONS.md | 1 + docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 15 +- docs/guides/CONDITIONS_AND_ROUTERS.md | 82 ++++ docs/reference/YAML.md | 9 +- examples/v1/README.md | 1 + examples/v1/router.yaml | 43 ++ schemas/workflow.schema.json | 53 +++ xtask/src/acceptance.rs | 61 ++- 22 files changed, 1005 insertions(+), 61 deletions(-) create mode 100644 docs/adr/0010-typed-routing-and-durable-decisions.md create mode 100644 docs/guides/CONDITIONS_AND_ROUTERS.md create mode 100644 examples/v1/router.yaml diff --git a/README.md b/README.md index 1f5d9fc..09b3599 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ For confidential workflow history, use [Sensitive-state encryption](docs/guides/ For environment, mounted-file, and policy-gated process credentials, use [Secret references](docs/guides/SECRET_REFERENCES.md). For bounded independent branches and working-memory conflict rules, use [Deterministic parallel tasks](docs/guides/PARALLEL_TASKS.md). For bounded static task expansion and child-level recovery, use [Matrix and foreach tasks](docs/guides/MATRIX_AND_FOREACH.md). +For typed branching and durable decisions, use [Conditions and routers](docs/guides/CONDITIONS_AND_ROUTERS.md). ## Safety boundary diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index ac63e36..1b786d2 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -29,6 +29,8 @@ pub struct CompiledTask { #[serde(default, skip_serializing_if = "Option::is_none")] pub expansion: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub route_guards: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub memory_writes: Vec, pub when: Option, pub vars: JsonMap, @@ -48,12 +50,34 @@ pub struct CompiledExpansion { pub bindings: JsonMap, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledRouter { + pub select: String, + pub cases: Vec, + pub default: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledRouteCase { + pub equals: Value, + pub tasks: Vec, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RouteGuard { + pub router: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "kind", content = "name")] pub enum TaskUse { Action(String), Agent(String), Aggregate(Vec), + Router(CompiledRouter), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -161,6 +185,17 @@ fn expand_tasks( ) -> Vec { let mut expanded = Vec::new(); for (position, task) in workflow.spec.tasks.iter().enumerate() { + if task.route.is_some() && (task.foreach.is_some() || task.matrix.is_some()) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("router task `{}` cannot be expanded", task.id), + ) + .with_path(format!("spec.tasks[{position}]")), + ); + continue; + } let bindings = match (&task.foreach, &task.matrix) { (None, None) => { expanded.push(ExpandedTaskDefinition { @@ -355,6 +390,7 @@ fn expand_tasks( aggregate.needs.clone_from(&children); aggregate.foreach = None; aggregate.matrix = None; + aggregate.route = None; aggregate.memory_writes.clear(); aggregate.when = None; aggregate.vars.clear(); @@ -414,7 +450,45 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result { TaskUse::Action(name) @@ -442,10 +516,10 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + TaskUse::Agent(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) => JsonMap::new(), }; input.extend(task.input.clone()); - let memory_writes = if matches!(task_use, TaskUse::Aggregate(_)) { + let memory_writes = if matches!(task_use, TaskUse::Aggregate(_) | TaskUse::Router(_)) { Vec::new() } else { task_memory_writes( @@ -465,7 +539,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + TaskUse::Action(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) => JsonMap::new(), }; vars.extend(task.vars.clone()); declaration_order.push(task.id.clone()); @@ -477,6 +551,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result Result PlanPredictability::FullyPredictable, + TaskUse::Router(_) => PlanPredictability::FullyPredictable, }; } let predictability = tasks @@ -647,7 +724,7 @@ fn task_memory_writes( .actions .get(name) .is_some_and(|action| action.kind == ActionKind::MemoryWrite), - TaskUse::Agent(_) | TaskUse::Aggregate(_) => false, + TaskUse::Agent(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) => false, }; if !is_memory_write { if !declared.is_empty() { @@ -709,6 +786,169 @@ fn task_memory_writes( writes.into_iter().collect() } +fn validate_routers( + tasks: &mut BTreeMap, + source_positions: &BTreeMap, + file: &str, + diagnostics: &mut Vec, +) { + let routers = tasks + .iter() + .filter_map(|(id, task)| match &task.uses { + TaskUse::Router(router) => Some((id.clone(), router.clone(), task.needs.clone())), + _ => None, + }) + .collect::>(); + for (router_id, router, needs) in routers { + let position = source_positions + .get(&router_id) + .copied() + .unwrap_or_default(); + let route_path = format!("spec.tasks[{position}].route"); + let trimmed = router.select.trim(); + let exact_template = trimmed.starts_with("${{") + && trimmed + .get(3..) + .and_then(|value| value.find("}}")) + .is_some_and(|closing| closing + 3 == trimmed.len() - 2); + if !exact_template { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::InvalidTemplate, + file, + format!("router task `{router_id}` select must be one exact typed template"), + ) + .with_path(format!("{route_path}.select")), + ); + } else if let Err(error) = validate_expression(&router.select) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::InvalidTemplate, + file, + format!("router task `{router_id}`: {error}"), + ) + .with_path(format!("{route_path}.select")), + ); + } + for reference in referenced_tasks(&router.select) { + if !tasks.contains_key(&reference) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::MissingReference, + file, + format!( + "router task `{router_id}` selector refers to unknown task `{reference}`" + ), + ) + .with_path(format!("{route_path}.select")), + ); + } else if !needs.contains(&reference) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::InvalidTemplate, + file, + format!( + "router task `{router_id}` must declare `{reference}` in needs before selecting from its output" + ), + ) + .with_path(format!("{route_path}.select")), + ); + } + } + if router.cases.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("router task `{router_id}` requires at least one case"), + ) + .with_path(format!("{route_path}.cases")), + ); + } + for (index, case) in router.cases.iter().enumerate() { + if case.tasks.is_empty() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("router task `{router_id}` case {index} has no destinations"), + ) + .with_path(format!("{route_path}.cases[{index}].tasks")), + ); + } + if router.cases[..index] + .iter() + .any(|previous| previous.equals == case.equals) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "router task `{router_id}` has duplicate typed case value {}", + case.equals + ), + ) + .with_path(format!("{route_path}.cases[{index}].equals")), + ); + } + } + + let mut destinations = BTreeSet::new(); + for destination in router + .cases + .iter() + .flat_map(|case| case.tasks.iter()) + .chain(router.default.iter()) + { + if !destinations.insert(destination.clone()) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "router task `{router_id}` destination `{destination}` is declared more than once" + ), + ) + .with_path(route_path.clone()), + ); + continue; + } + let Some(target) = tasks.get(destination) else { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::MissingReference, + file, + format!( + "router task `{router_id}` refers to unknown destination `{destination}`" + ), + ) + .with_path(route_path.clone()), + ); + continue; + }; + if !target.needs.contains(&router_id) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "routed task `{destination}` must declare router `{router_id}` in needs" + ), + ) + .with_path(route_path.clone()), + ); + continue; + } + if let Some(target) = tasks.get_mut(destination) { + target.route_guards.push(RouteGuard { + router: router_id.clone(), + }); + } + } + } +} + fn validate_parallel_memory_writes( workflow: &Workflow, order: &[String], @@ -853,7 +1093,7 @@ fn plan_requirements( predictability: task.predictability, }] } - TaskUse::Aggregate(_) => Vec::new(), + TaskUse::Aggregate(_) | TaskUse::Router(_) => Vec::new(), }) .collect(); PlanRequirements { @@ -1600,6 +1840,89 @@ spec: })); } + #[test] + fn typed_router_cases_compile_to_explicit_destination_guards() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: typed-router } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - { id: decide, uses: "action:assign", with: { route: ship } } + - id: route + uses: router + needs: [decide] + route: + select: "${{ tasks.decide.output.output.route }}" + cases: + - { equals: ship, tasks: [ship] } + - { equals: 1, tasks: [numeric] } + default: [hold] + - { id: ship, uses: "action:assign", needs: [route] } + - { id: numeric, uses: "action:assign", needs: [route] } + - { id: hold, uses: "action:assign", needs: [route] } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("compiles"); + let router = match &plan.tasks["route"].uses { + TaskUse::Router(router) => router, + other => panic!("expected router, got {other:?}"), + }; + assert_eq!(router.cases[0].equals, "ship"); + assert_eq!(router.cases[1].equals, 1); + assert_eq!( + plan.tasks["ship"].route_guards, + [RouteGuard { + router: "route".to_owned() + }] + ); + assert_eq!(plan.order, ["decide", "route", "ship", "numeric", "hold"]); + } + + #[test] + fn router_rejects_ambiguous_cases_and_implicit_dependencies() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: invalid-router } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - { id: decide, uses: "action:assign", with: { route: ship } } + - id: route + uses: router + needs: [decide] + route: + select: "${{ tasks.decide.output.output.route }}" + cases: + - { equals: ship, tasks: [ship] } + - { equals: ship, tasks: [ship] } + - { id: ship, uses: "action:assign" } +"#, + ); + let diagnostics = compile(&workflow, "fixture.yaml").expect_err("router rejected"); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("duplicate typed case value")) + ); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("must declare router `route` in needs") + })); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("destination `ship` is declared more than once") + })); + } + #[test] fn parallel_memory_writes_are_inferred_and_conflicts_are_rejected() { let workflow = parse( diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index 37f6764..708f106 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -424,6 +424,8 @@ pub struct TaskDefinition { pub foreach: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub matrix: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub route: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub memory_writes: Vec, #[serde(default)] @@ -471,6 +473,22 @@ pub struct MatrixDefinition { pub max_items: usize, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RouteDefinition { + pub select: String, + pub cases: Vec, + #[serde(default)] + pub default: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RouteCaseDefinition { + pub equals: Value, + pub tasks: Vec, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RetryDefinition { diff --git a/crates/agentctl-core/tests/compatibility.rs b/crates/agentctl-core/tests/compatibility.rs index b1e3bbe..d718479 100644 --- a/crates/agentctl-core/tests/compatibility.rs +++ b/crates/agentctl-core/tests/compatibility.rs @@ -39,6 +39,8 @@ fn typescript_assign_fixture_translates_to_the_language_neutral_contract() { assert_eq!(expected.task.use_kind, "action"); assert_eq!(reference, &expected.task.reference); } - TaskUse::Agent(_) | TaskUse::Aggregate(_) => panic!("expected action task"), + TaskUse::Agent(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) => { + panic!("expected action task") + } } } diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index b1d3be9..2777f9b 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -782,26 +782,28 @@ impl Runtime { &trace_id, )?; for (task, terminal) in source_tasks { - self.store.transition_task( - &replay_id, - &task.task_id, - TaskState::Ready, - None, - None, - None, - self.clock.now(), - &trace_id, - )?; - self.store.transition_task( - &replay_id, - &task.task_id, - TaskState::Running, - None, - None, - None, - self.clock.now(), - &trace_id, - )?; + if terminal != TaskState::Skipped { + self.store.transition_task( + &replay_id, + &task.task_id, + TaskState::Ready, + None, + None, + None, + self.clock.now(), + &trace_id, + )?; + self.store.transition_task( + &replay_id, + &task.task_id, + TaskState::Running, + None, + None, + None, + self.clock.now(), + &trace_id, + )?; + } self.store.transition_task( &replay_id, &task.task_id, @@ -2436,7 +2438,6 @@ impl Runtime { }); } - let context = context_for(&run, &tasks)?; let mut prepared_pending = false; for task_id in &run.plan.order { let Some(record) = tasks.iter().find(|record| &record.task_id == task_id) else { @@ -2482,14 +2483,13 @@ impl Runtime { prepared_pending = true; continue; } - if let Some(condition) = &task.when - && !evaluate_when(condition, &context)? - { + let context = context_for_task(&run, &tasks, task)?; + if let Some(decision) = route_skip_decision(task, &context)? { self.store.transition_task( run_id, task_id, TaskState::Skipped, - Some(&serde_json::json!({"reason": "when condition was false"})), + Some(&decision), None, None, self.clock.now(), @@ -2498,11 +2498,35 @@ impl Runtime { prepared_pending = true; continue; } + let condition_decision = if let Some(condition) = &task.when { + let result = evaluate_when(condition, &context)?; + let decision = condition_decision(condition, result, &context)?; + if !result { + self.store.transition_task( + run_id, + task_id, + TaskState::Skipped, + Some(&serde_json::json!({ + "reason": "when condition was false", + "condition": decision["condition"], + })), + None, + None, + self.clock.now(), + trace_id, + )?; + prepared_pending = true; + continue; + } + Some(decision) + } else { + None + }; self.store.transition_task( run_id, task_id, TaskState::Ready, - None, + condition_decision.as_ref(), None, None, self.clock.now(), @@ -3004,15 +3028,13 @@ impl Runtime { .map(|record| record.state) .ok_or_else(|| RuntimeError::InvalidState(format!("task `{}` missing", task.id)))?; if ready_state == TaskState::Pending { - let context = context_for(&run, &tasks)?; - if let Some(condition) = &task.when - && !evaluate_when(condition, &context)? - { + let context = context_for_task(&run, &tasks, task)?; + if let Some(decision) = route_skip_decision(task, &context)? { self.store.transition_task( run_id, &task.id, TaskState::Skipped, - Some(&serde_json::json!({"reason": "when condition was false"})), + Some(&decision), None, None, self.clock.now(), @@ -3020,11 +3042,34 @@ impl Runtime { )?; continue; } + let condition_decision = if let Some(condition) = &task.when { + let result = evaluate_when(condition, &context)?; + let decision = condition_decision(condition, result, &context)?; + if !result { + self.store.transition_task( + run_id, + &task.id, + TaskState::Skipped, + Some(&serde_json::json!({ + "reason": "when condition was false", + "condition": decision["condition"], + })), + None, + None, + self.clock.now(), + trace_id, + )?; + continue; + } + Some(decision) + } else { + None + }; self.store.transition_task( run_id, &task.id, TaskState::Ready, - None, + condition_decision.as_ref(), None, None, self.clock.now(), @@ -3312,12 +3357,7 @@ impl Runtime { cancellation: &CancellationToken, ) -> Result { let tasks = self.store.list_tasks(&run.run_id)?; - let mut context = context_for(run, &tasks)?; - context.vars = task - .vars - .iter() - .map(|(name, value)| render(value, &context).map(|value| (name.clone(), value))) - .collect::, _>>()?; + let context = context_for_task(run, &tasks, task)?; let raw_input = serde_json::to_value(&task.input)?; let input = render(&raw_input, &context)?; match &task.uses { @@ -3403,6 +3443,20 @@ impl Runtime { memory: None, }) } + TaskUse::Router(router) => { + let selected = render(&Value::String(router.select.clone()), &context)?; + let matched = router.cases.iter().find(|case| case.equals == selected); + let destinations = + matched.map_or_else(|| router.default.clone(), |case| case.tasks.clone()); + Ok(TaskExecution::Complete { + output: serde_json::json!({ + "selected": selected, + "matched": matched.is_some(), + "destinations": destinations, + }), + memory: None, + }) + } } } @@ -4874,6 +4928,19 @@ fn task_output_schema(workflow: &Workflow, task: &agentctl_core::CompiledTask) - "properties": {"items": {"type": "array"}}, "additionalProperties": false })), + TaskUse::Router(_) => Some(serde_json::json!({ + "type": "object", + "required": ["selected", "matched", "destinations"], + "properties": { + "selected": {}, + "matched": {"type": "boolean"}, + "destinations": { + "type": "array", + "items": {"type": "string"} + } + }, + "additionalProperties": false + })), }) } @@ -4963,6 +5030,11 @@ fn task_definition_fingerprint( "task": task, "children": children, }), + TaskUse::Router(router) => serde_json::json!({ + "kind": "router", + "task": task, + "router": router, + }), }; versioned_json_digest(&serde_json::json!({ "formatVersion": 1, @@ -5702,6 +5774,77 @@ fn context_for( }) } +fn context_for_task( + run: &agentctl_store::RunRecord, + tasks: &[TaskRecord], + task: &agentctl_core::CompiledTask, +) -> Result { + let mut context = context_for(run, tasks)?; + context.vars = task + .vars + .iter() + .map(|(name, value)| render(value, &context).map(|value| (name.clone(), value))) + .collect::, _>>()?; + Ok(context) +} + +fn condition_decision( + expression: &str, + result: bool, + context: &EvalContext, +) -> Result { + let context_digest = versioned_json_digest(&serde_json::json!({ + "inputs": context.inputs, + "vars": context.vars, + "memory": context.memory, + "tasks": context.tasks, + }))?; + Ok(serde_json::json!({ + "condition": { + "expression": expression, + "contextDigest": context_digest, + "result": result, + } + })) +} + +fn route_skip_decision( + task: &agentctl_core::CompiledTask, + context: &EvalContext, +) -> Result, RuntimeError> { + for guard in &task.route_guards { + let decision = context.tasks.get(&guard.router).ok_or_else(|| { + RuntimeError::InvalidState(format!( + "router `{}` has no durable output for task `{}`", + guard.router, task.id + )) + })?; + let destinations = decision + .get("destinations") + .and_then(Value::as_array) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "router `{}` output has no destinations", + guard.router + )) + })?; + if !destinations + .iter() + .any(|destination| destination.as_str() == Some(task.id.as_str())) + { + return Ok(Some(serde_json::json!({ + "reason": "route not selected", + "route": { + "router": guard.router, + "selected": decision.get("selected"), + "matched": decision.get("matched"), + } + }))); + } + } + Ok(None) +} + fn collect_outputs( run: &agentctl_store::RunRecord, tasks: &[TaskRecord], @@ -9474,6 +9617,166 @@ spec: ); } + #[tokio::test] + async fn typed_router_persists_decisions_and_recovers_when_the_route_changes() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: typed-routing } +spec: + policy: { approval: never } + actions: + assign: { kind: builtin.assign } + tasks: + - { id: decide, uses: "action:assign", with: { route: ship } } + - id: route + uses: router + needs: [decide] + route: + select: "${{ tasks.decide.output.output.route }}" + cases: + - { equals: ship, tasks: [ship] } + default: [hold] + - id: ship + uses: action:assign + needs: [route] + vars: { enabled: true } + when: "${{ vars.enabled == true }}" + with: { result: shipped } + - { id: hold, uses: "action:assign", needs: [route], with: { result: held } } +"#; + let (source_workflow, source_plan) = compile_fixture(source); + let source_outcome = runtime + .start( + &source_workflow, + &source_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("source route"); + assert_eq!(source_outcome.state, RunState::Succeeded); + let source_tasks = store + .list_tasks(&source_outcome.run_id) + .expect("source tasks"); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == "route") + .and_then(|task| task.output.as_ref()) + .expect("route output")["destinations"], + serde_json::json!(["ship"]) + ); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == "hold") + .expect("hold") + .state, + TaskState::Skipped + ); + let condition_audit = store + .audit_events(&source_outcome.run_id) + .expect("audit") + .into_iter() + .find(|event| { + event.event_type == "task.transition" + && event.task_id.as_deref() == Some("ship") + && event.payload["to"] == "ready" + }) + .expect("condition decision audit"); + assert_eq!( + condition_audit.payload["decision"]["condition"]["result"], + true + ); + assert!( + condition_audit.payload["decision"]["condition"]["contextDigest"] + .as_str() + .is_some_and(|digest| digest.starts_with("sha256:v1:")) + ); + + let retry_plan = runtime + .plan_retry( + &source_outcome.run_id, + &source_workflow, + &source_plan, + &["route".to_owned()], + false, + true, + ) + .expect("router retry plan"); + assert_eq!(retry_plan.reused_tasks, ["decide"]); + let retried = runtime + .retry( + &source_workflow, + &source_plan, + retry_plan, + Some("re-evaluate typed route"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("router retry"); + assert_eq!(retried.state, RunState::Succeeded); + + let target = source.replacen("with: { route: ship }", "with: { route: hold }", 1); + let (target_workflow, target_plan) = compile_fixture(&target); + let repair_plan = runtime + .plan_repair( + &source_outcome.run_id, + &target_workflow, + &target_plan, + &["decide".to_owned()], + true, + ) + .expect("route repair plan"); + assert!(repair_plan.compatible, "{:?}", repair_plan.blocked_reuse); + let repaired = runtime + .repair( + &target_workflow, + &target_plan, + repair_plan, + Some("change typed route"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("route repair"); + assert_eq!(repaired.state, RunState::Succeeded); + let repaired_tasks = store.list_tasks(&repaired.run_id).expect("repaired tasks"); + assert_eq!( + repaired_tasks + .iter() + .find(|task| task.task_id == "ship") + .expect("ship") + .state, + TaskState::Skipped + ); + assert_eq!( + repaired_tasks + .iter() + .find(|task| task.task_id == "hold") + .expect("hold") + .state, + TaskState::Succeeded + ); + let replay = runtime + .replay(&repaired.run_id) + .await + .expect("route replay"); + assert_eq!(replay.state, RunState::Succeeded); + assert!( + store + .list_effects(&replay.run_id) + .expect("replay effects") + .is_empty() + ); + } + #[tokio::test] async fn task_output_contract_failure_is_durable() { let directory = tempdir().expect("tempdir"); diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index a15d497..28fec60 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -1735,7 +1735,12 @@ impl SqliteStore { "task.transition", Some(task_id), trace_id, - &serde_json::json!({"from": current, "to": next, "error": error}), + &serde_json::json!({ + "from": current, + "to": next, + "error": error, + "decision": output, + }), now, &self.protection, )?; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 906f33a..529b21f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,9 +34,11 @@ sessions remain task-local. See ADR 0008 and Static foreach lists and matrix axes compile into ordinary namespaced child tasks followed by a pure aggregate. Their IDs, bindings, attempts, outputs, -and recovery lineage use the same durable task model as authored nodes. Loops, -routers, sub-workflows, handlers, compensation execution, and event triggers -still require their own explicit state and recovery contracts. The DSL carries +and recovery lineage use the same durable task model as authored nodes. Typed +routers are pure tasks whose enumerated destination guards compile into the +graph; condition and route decisions are durable and replayable. Loops, +sub-workflows, handlers, compensation execution, and event triggers still +require their own explicit state and recovery contracts. The DSL carries optional compensation metadata on a tool contract, but the runtime does not execute compensation. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 8a2ae4f..b25dbe8 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -22,4 +22,4 @@ Legacy workflows depending on packs, broad built-in tool profiles, remote MCP/A2 ## Deferred product decisions -Loops, routers, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. +Loops, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. diff --git a/docs/DSL.md b/docs/DSL.md index 59eb97c..c161c55 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,10 +2,16 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` either `action:` or `agent:`, declares `needs`, optional bounded `foreach` or `matrix` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach` or `matrix` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. +`when` decisions retain the expression, boolean result, and a digest of the +evaluated context in durable task/audit state. A `router` selects one exact +typed template, compares it with type-sensitive enumerated cases, and records +the selected value and explicit destinations. Unselected destinations are +durably skipped. See [Conditions and routers](guides/CONDITIONS_AND_ROUTERS.md). + Task output is JSON. Built-in actions own an object contract, agents can declare provider-enforced `structuredOutput`, and a task can override the complete contract with `outputSchema`. The compiler validates schemas; the runtime validates completed and selectively reused values. Providers, action environments, and protocol headers use secret references: @@ -28,4 +34,4 @@ select the visible child IDs. See [Matrix and foreach tasks](guides/MATRIX_AND_F The parser translates a limited unversioned `playbook:` document and emits a migration warning. Use `agentctl migrate old.yaml --write new.yaml`. Legacy pack-backed, MCP, A2A, provider-specific, and broad module configurations need manual migration; see [Migrating from TypeScript](MIGRATING_FROM_TYPESCRIPT.md). -Not implemented in v1alpha1: routers, loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. +Not implemented in v1alpha1: loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index b29aac6..52a94b2 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -37,4 +37,10 @@ fingerprint, output, retry/repair identity, and replay record. The parent is a pure aggregate task that records child IDs, states, outputs, and errors in stable expansion order. +Condition transitions retain the expression, boolean result, and a canonical +digest of the evaluated inputs, variables, memory, and dependency outputs. +Pure router tasks retain their typed selected value and enumerated destination +IDs. Skipped branch records are copied directly by recorded replay and never +pass through a running state. + The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 3bdc9c9..18ba122 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -23,7 +23,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an These are useful extensions but are not required by the product thesis. They need new deterministic state and compatibility contracts before implementation: -- loops; routers; sub-workflows; compensation execution; +- loops; sub-workflows; compensation execution; - structured agent teams and handoffs; - model token streaming into CLI/workflow state; - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; @@ -46,6 +46,9 @@ These are useful extensions but are not required by the product thesis. They nee - Foreach and matrix expansion accepts only static workflow values, requires `maxItems`, and is capped at 256 children. Runtime or model-controlled graph growth is not supported. +- Conditions support typed paths, equality, and `not`; routers support exact + typed selectors and enumerated destinations. Arbitrary expressions, implicit + dependencies, and model-owned hidden routing are rejected. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. diff --git a/docs/adr/0010-typed-routing-and-durable-decisions.md b/docs/adr/0010-typed-routing-and-durable-decisions.md new file mode 100644 index 0000000..9bc3e83 --- /dev/null +++ b/docs/adr/0010-typed-routing-and-durable-decisions.md @@ -0,0 +1,24 @@ +# ADR 0010: typed routing and durable decisions + +Status: accepted + +## Decision + +Conditions remain a constrained path, equality, and `not` language. Router +tasks are pure compiled nodes with one exact typed selector, unique JSON case +values, enumerated destination tasks, and optional default destinations. +Every destination declares the router as a dependency. + +Condition transitions retain the expression, result, and evaluated-context +digest. Router output retains the selected JSON value, whether a case matched, +and the chosen destination IDs. Unselected destinations become skipped with an +explicit route decision. + +## Consequences + +- Arbitrary code and hidden model-owned routing remain impossible. +- JSON case comparison is type-sensitive. +- Changed selector dependencies invalidate router reuse during repair. +- Skipped tasks have a control-flow output contract separate from their normal + execution output contract. +- Retry and recorded replay preserve the same visible decision boundaries. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 76ecf3d..e8eaba6 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -76,7 +76,8 @@ cargo xtask acceptance-container | Secret references | environment compatibility, file bounds/missing/symlink containment, process allowlist/timeout/output/cancellation, zeroizing values, adapter redaction, and raw-database absence tests passed | packaged CLI scenario 32 and the 12-stage verification gate passed | verified | | Parallel scheduling | overlap, caps, conflicts, ordered atomic commits, approvals, cancellation, retry, repair, and replay tests passed | packaged CLI scenario 33 and OCI parallel run/replay passed | deterministic verified; live pending | | Foreach/matrix | compiler bounds/identity tests and runtime partial-failure, child retry, sibling reuse, aggregation, and replay tests passed | packaged CLI scenario 34 pending | deterministic in progress | -| Conditions/loops/sub-workflows | pending | pending | open | +| Conditions/routers | compiler typed-case/guard failures and runtime durable condition, route, retry, changed-input repair, and skipped replay tests passed | packaged CLI scenario 35 pending | deterministic in progress | +| Loops/sub-workflows | pending | pending | open | | Compensation/handoffs/streaming | pending | pending | open | | MCP/A2A resilience | pending | pending | open | | Packs/trust/extensions | pending | pending | open | diff --git a/docs/execution/DECISIONS.md b/docs/execution/DECISIONS.md index 7092dfe..af50fe3 100644 --- a/docs/execution/DECISIONS.md +++ b/docs/execution/DECISIONS.md @@ -11,5 +11,6 @@ | [0007](../adr/0007-generic-oci-step-contract.md) | Generic OCI step contract | accepted | Non-root/read-only image; mounted config/workspace/state/artifacts. | | [0008](../adr/0008-deterministic-parallel-batches.md) | Deterministic parallel batches | accepted | Bounded overlap, isolated snapshots, declared writes, and atomic plan-order commits. | | [0009](../adr/0009-bounded-static-task-expansion.md) | Bounded static task expansion | accepted | Stable child tasks and ordered aggregates prevent model-controlled graph growth. | +| [0010](../adr/0010-typed-routing-and-durable-decisions.md) | Typed routing and durable decisions | accepted | Pure enumerated routers and hashed condition contexts make branching inspectable and replayable. | These decisions resolve the researched patterns in [LANDSCAPE.md](../research/LANDSCAPE.md). No unsafe code or distributed control plane ADR is required because neither exists. diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md index fa51890..d71444c 100644 --- a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -44,6 +44,7 @@ This inventory is enforced by `cargo xtask examples-verify`. The default command | `examples/v1/parallel.yaml` | Deterministic parallel batch | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Atomic ordered memory merge | Canonical | passed | | `examples/v1/policy-denial.yaml` | Denied mutation | deterministic | policy failure | 0 | 0 | Canonical expected failure | N/A | N/A | N/A | No mutation | JSON error | passed | | `examples/v1/reusable-pack.yaml` | Native reusable pack consumer | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Pack digest | Canonical | passed | +| `examples/v1/router.yaml` | Typed deterministic route selection | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Decision and skipped branch | passed | | `examples/v1/secret-reference.yaml` | Environment reference contract | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Secret-safe live gate | live passed | | `examples/v1/working-memory.yaml` | Working-memory update | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | | `fixtures/compat/v0/assign.playbook.yaml` | Language-neutral TypeScript compatibility fixture | legacy translator | success | 0 | 0 | Compatibility test | N/A | N/A | N/A | N/A | Oracle contract | passed | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index 55a4fcd..3c7bf1a 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -45,7 +45,7 @@ complete, every entry must have exactly one final disposition: | BUD-001 | Resource and cost budgets | open | implemented | | SCH-001 | Deterministic parallel execution | in progress | implemented | | DYN-001 | Foreach and matrix | in progress | implemented | -| COND-001 | Conditions and routers | open | implemented | +| COND-001 | Conditions and routers | in progress | implemented | | LOOP-001 | Bounded loops | open | implemented | | SUB-001 | Sub-workflows | open | implemented | | COMP-001 | Compensation | open | implemented | @@ -261,8 +261,8 @@ complete, every entry must have exactly one final disposition: ### COND-001: Typed conditions and routers -- Current behavior: constrained `not` and equality conditions exist, but there - is no explicit router or complete skipped-output contract. +- Current behavior: constrained typed conditions and explicit pure router tasks + persist decisions and skip only enumerated destinations. - User impact: nontrivial deterministic branching is awkward. - Security or durability impact: expanding to arbitrary expressions would add code execution and ambiguous dependencies. @@ -277,7 +277,14 @@ complete, every entry must have exactly one final disposition: - Examples: structured agent output routed to deterministic branches. - Live evidence: one structured-output routing scenario. - Documentation: expression grammar and skip semantics. -- Final disposition: pending implementation evidence. +- Final disposition: implemented. Conditions use the constrained typed + evaluator and retain expression, context digest, and result. Routers compare + one exact typed selector against unique JSON cases, record selected value and + destinations, and durably skip unselected branches. Deterministic + verification covers strict typing, malformed routes, local vars, plan + guards, retry, changed-decision repair, skipped-task replay, and zero replay + effects. Program state remains in progress until the structured OpenAI + routing scenario executes. ### LOOP-001: Bounded loops diff --git a/docs/guides/CONDITIONS_AND_ROUTERS.md b/docs/guides/CONDITIONS_AND_ROUTERS.md new file mode 100644 index 0000000..6fb2760 --- /dev/null +++ b/docs/guides/CONDITIONS_AND_ROUTERS.md @@ -0,0 +1,82 @@ +# Conditions and routers + +Conditions and routers are deterministic control-flow nodes. They use the +constrained template evaluator and cannot execute code. + +## Conditions + +`when` accepts one path, optional `not`, or equality against a JSON value: + +```yaml +vars: + enabled: true +when: "${{ vars.enabled == true }}" +``` + +Equality is type-sensitive. Missing paths fail instead of becoming false. +Explicit `null` is a value and is false when used directly. A false condition +sets the task to `skipped` with this durable output shape: + +```json +{ + "reason": "when condition was false", + "condition": { + "expression": "${{ vars.enabled == true }}", + "contextDigest": "sha256:v1:...", + "result": false + } +} +``` + +For a true condition, the same decision is retained in the transition audit +before the normal task output replaces the temporary decision value. + +## Routers + +A router is a pure task with an exact typed selector and enumerated +destinations: + +```yaml +- id: route + uses: router + needs: [classify] + route: + select: "${{ tasks.classify.output.route }}" + cases: + - equals: approve + tasks: [approved] + - equals: reject + tasks: [rejected] + default: [manual] +``` + +Every destination must depend directly on the router. A destination may appear +only once. Duplicate typed case values, unknown destinations, implicit +dependencies, interpolated selectors, and missing cases fail compilation. +Case comparison uses JSON identity, so the number `1` and string `"1"` are +different. + +The router output is: + +```json +{ + "selected": "approve", + "matched": true, + "destinations": ["approved"] +} +``` + +Unselected destinations become `skipped` with the router ID and selected value +in their durable output. Their normal output contract is not evaluated because +they did not execute. A downstream task whose dependency was skipped is also +skipped. + +## Recovery + +The selector's dependency outputs and task variables participate in the +resolved-input digest. Changing an upstream classification invalidates the +router and its guarded descendants during repair. Terminal retry can restart a +router explicitly, and recorded replay copies route and skip decisions without +dispatching effects. + +See [`examples/v1/router.yaml`](../../examples/v1/router.yaml). diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index df86e37..b5cb100 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -36,13 +36,15 @@ Unknown fields fail. Documents, ordinary input files, packs, direct reads, exist ## Tasks -Each task requires `id` and `uses`. `uses` is `action:name` or `agent:name`. +Each task requires `id` and `uses`. `uses` is `action:name`, `agent:name`, or +`router`. | Field | Default | Validation | | --- | --- | --- | | `needs` | `[]` | Every ID must exist; cycles fail. | | `foreach` | none | Static typed `items`, binding `as`, and `maxItems`. Mutually exclusive with `matrix`; maximum 256 children. | | `matrix` | none | Static `axes` Cartesian product and `maxItems`. Axis names are template-safe identifiers; maximum 256 children. | +| `route` | required for `uses: router` | Exact typed `select`, unique typed cases, enumerated destinations, and optional default destinations. Every destination must depend on the router. | | `memoryWrites` | inferred or `[]` | Working-memory keys. Literal memory-write keys are inferred; templated keys require an explicit set. Unordered overlaps fail when concurrency is greater than one. | | `when` | true | Constrained boolean/equality expression. | | `vars` | `{}` | Task-local JSON values. | @@ -56,7 +58,7 @@ Ready tasks are selected in YAML declaration order up to `maxConcurrency`. They read isolated durable snapshots and commit in compiled order. There is no runtime or model-controlled expansion. Static `foreach` and `matrix` tasks compile to inspectable child tasks and a parent aggregate. There is no loop, -router, sub-workflow, handler, or separate parallel group in this version. +sub-workflow, handler, or separate parallel group in this version. ## Agents @@ -121,6 +123,7 @@ agentctl run examples/v1/dataflow.yaml --db /tmp/dataflow.db --output json --col ``` Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Matrix -and foreach](../guides/MATRIX_AND_FOREACH.md), [Secret +and foreach](../guides/MATRIX_AND_FOREACH.md), [Conditions and +routers](../guides/CONDITIONS_AND_ROUTERS.md), [Secret references](../guides/SECRET_REFERENCES.md), [Policies](../policies.md), [Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). diff --git a/examples/v1/README.md b/examples/v1/README.md index 0611949..6b65f9d 100644 --- a/examples/v1/README.md +++ b/examples/v1/README.md @@ -5,6 +5,7 @@ The deterministic examples are exercised by `cargo xtask verify` and never requi - `hello.yaml`: deterministic hello world and declared output. - `dataflow.yaml`: typed scalar/object templates. - `condition.yaml`: safe `when` equality and skipping. +- `router.yaml`: typed route selection, explicit destinations, and durable skip decisions. - `check-diff.yaml`: predictable file diff without mutation under `run --check --diff`. - `approval.yaml`: durable approval-gated workspace mutation. - `policy-denial.yaml`: an explicit tool-policy denial. diff --git a/examples/v1/router.yaml b/examples/v1/router.yaml new file mode 100644 index 0000000..8d13e5e --- /dev/null +++ b/examples/v1/router.yaml @@ -0,0 +1,43 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: typed-router +spec: + inputs: + selected: ship + outputs: + selected: "${{ tasks.route.output.selected }}" + ship: "${{ tasks.ship.output }}" + hold: "${{ tasks.hold.output }}" + policy: + approval: never + actions: + assign: + kind: builtin.assign + tasks: + - id: decide + uses: action:assign + with: + route: "${{ inputs.selected }}" + - id: route + uses: router + needs: [decide] + route: + select: "${{ tasks.decide.output.output.route }}" + cases: + - equals: ship + tasks: [ship] + default: [hold] + - id: ship + uses: action:assign + needs: [route] + vars: + enabled: true + when: "${{ vars.enabled == true }}" + with: + result: shipped + - id: hold + uses: action:assign + needs: [route] + with: + result: held diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index 7526db7..91ab1d1 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -722,6 +722,16 @@ } ] }, + "route": { + "anyOf": [ + { + "$ref": "#/$defs/RouteDefinition" + }, + { + "type": "null" + } + ] + }, "memoryWrites": { "type": "array", "items": { @@ -818,6 +828,49 @@ "axes" ] }, + "RouteDefinition": { + "type": "object", + "properties": { + "select": { + "type": "string" + }, + "cases": { + "type": "array", + "items": { + "$ref": "#/$defs/RouteCaseDefinition" + } + }, + "default": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + } + }, + "additionalProperties": false, + "required": [ + "select", + "cases" + ] + }, + "RouteCaseDefinition": { + "type": "object", + "properties": { + "equals": true, + "tasks": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "equals", + "tasks" + ] + }, "FailureBehavior": { "type": "string", "enum": [ diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 359e235..2228af7 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 34; +const ACCEPTANCE_SCENARIOS: usize = 35; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1431,6 +1431,65 @@ pub fn run(root: &Path) -> Result<()> { let matrix_replay_inspect = inspect(&binary, root, &matrix_db, matrix_replay_id)?; ensure!(array_len(&matrix_replay_inspect, "/data/effects")? == 0); + scenario( + 35, + "packaged CLI persists, inspects, and replays a typed route decision", + ); + let router = root.join("examples/v1/router.yaml"); + let router_plan = successful_json( + &binary, + root, + &strings([ + "plan", + path(&router)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&router_plan, "/data/tasks/route/uses/kind", "router")?; + ensure_eq( + &router_plan, + "/data/tasks/ship/routeGuards/0/router", + "route", + )?; + let router_db = directory.path().join("router.db"); + let router_run = successful_json(&binary, root, &run_args(&router, &router_db, root, &[]))?; + ensure_eq(&router_run, "/data/state", "succeeded")?; + ensure_eq(&router_run, "/data/output/selected", "ship")?; + ensure_eq(&router_run, "/data/output/ship/output/result", "shipped")?; + ensure_eq( + &router_run, + "/data/output/hold/reason", + "route not selected", + )?; + let router_run_id = string_at(&router_run, "/data/runId")?; + let router_inspect = inspect(&binary, root, &router_db, router_run_id)?; + ensure_eq( + &router_inspect, + "/data/tasks/3/output/route/router", + "route", + )?; + let router_replay = successful_json( + &binary, + root, + &strings([ + "replay", + router_run_id, + "--db", + path(&router_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&router_replay, "/data/state", "succeeded")?; + let router_replay_id = string_at(&router_replay, "/data/runId")?; + let router_replay_inspect = inspect(&binary, root, &router_db, router_replay_id)?; + ensure!(array_len(&router_replay_inspect, "/data/effects")? == 0); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } From d2cfe3dedf265218d3d943f6992b0a4a8a6082bb Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 22:47:57 +0530 Subject: [PATCH 14/44] feat: add bounded durable loops --- README.md | 1 + crates/agentctl-core/src/compiler.rs | 383 ++++++++++++++- crates/agentctl-core/src/dsl.rs | 55 +++ crates/agentctl-core/src/template.rs | 73 ++- crates/agentctl-core/tests/compatibility.rs | 5 +- crates/agentctl-runtime/src/lib.rs | 437 +++++++++++++++++- crates/agentctl-runtime/src/process.rs | 2 +- docs/ARCHITECTURE.md | 9 +- docs/COMPATIBILITY.md | 6 +- docs/DSL.md | 11 +- docs/DURABLE_EXECUTION.md | 5 + docs/LIMITATIONS.md | 12 +- ...005-narrow-v1-scheduling-and-extensions.md | 9 +- ...010-typed-routing-and-durable-decisions.md | 2 +- .../0011-bounded-loops-as-static-graphs.md | 25 + docs/execution/COMPLETENESS_VERIFICATION.md | 7 +- docs/execution/DECISIONS.md | 1 + docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 19 +- docs/guides/BOUNDED_LOOPS.md | 79 ++++ docs/guides/CONDITIONS_AND_ROUTERS.md | 6 +- docs/reference/YAML.md | 9 +- examples/v1/README.md | 1 + examples/v1/loop.yaml | 24 + schemas/workflow.schema.json | 31 ++ xtask/src/acceptance.rs | 65 ++- 26 files changed, 1216 insertions(+), 62 deletions(-) create mode 100644 docs/adr/0011-bounded-loops-as-static-graphs.md create mode 100644 docs/guides/BOUNDED_LOOPS.md create mode 100644 examples/v1/loop.yaml diff --git a/README.md b/README.md index 09b3599..276f40e 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ For environment, mounted-file, and policy-gated process credentials, use [Secret For bounded independent branches and working-memory conflict rules, use [Deterministic parallel tasks](docs/guides/PARALLEL_TASKS.md). For bounded static task expansion and child-level recovery, use [Matrix and foreach tasks](docs/guides/MATRIX_AND_FOREACH.md). For typed branching and durable decisions, use [Conditions and routers](docs/guides/CONDITIONS_AND_ROUTERS.md). +For iterative work with a hard execution ceiling and iteration-level recovery, use [Bounded loops](docs/guides/BOUNDED_LOOPS.md). ## Safety boundary diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index 1b786d2..73fcdb5 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -7,8 +7,8 @@ use sha2::{Digest, Sha256}; use crate::PLAN_FORMAT_VERSION; use crate::diagnostic::{Diagnostic, DiagnosticCode}; use crate::dsl::{ - ActionKind, EffectClass, Idempotency, JsonMap, MAX_EXPANSION_ITEMS, ProviderKind, - RetryDefinition, TaskDefinition, ToolKind, Workflow, + ActionKind, EffectClass, Idempotency, JsonMap, MAX_EXPANSION_ITEMS, MAX_LOOP_ITERATIONS, + ProviderKind, RetryDefinition, TaskDefinition, ToolKind, Workflow, }; use crate::template::{TemplateError, referenced_tasks, validate_expression}; @@ -71,6 +71,13 @@ pub struct RouteGuard { pub router: String, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledLoopAggregate { + pub children: Vec, + pub condition: String, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "kind", content = "name")] pub enum TaskUse { @@ -78,6 +85,7 @@ pub enum TaskUse { Agent(String), Aggregate(Vec), Router(CompiledRouter), + LoopAggregate(CompiledLoopAggregate), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -175,7 +183,7 @@ struct ExpandedTaskDefinition { definition: TaskDefinition, source_position: usize, expansion: Option, - aggregate_children: Option>, + synthetic_use: Option, } fn expand_tasks( @@ -185,6 +193,161 @@ fn expand_tasks( ) -> Vec { let mut expanded = Vec::new(); for (position, task) in workflow.spec.tasks.iter().enumerate() { + if let Some(loop_definition) = &task.loop_definition { + let loop_path = format!("spec.tasks[{position}].loop"); + if task.foreach.is_some() || task.matrix.is_some() || task.route.is_some() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "loop task `{}` cannot also declare foreach, matrix, or route", + task.id + ), + ) + .with_path(loop_path), + ); + continue; + } + if task.uses == "router" { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("router task `{}` cannot be a loop body", task.id), + ) + .with_path(format!("spec.tasks[{position}].uses")), + ); + continue; + } + if task.when.is_some() { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "loop task `{}` uses loop.while as its iteration guard and cannot also declare when", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}].when")), + ); + continue; + } + if task.vars.contains_key("loopIndex") || task.vars.contains_key("loopPrevious") { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("loop task `{}` bindings conflict with task vars", task.id), + ) + .with_path(format!("spec.tasks[{position}].vars")), + ); + continue; + } + if loop_definition.max_iterations == 0 + || loop_definition.max_iterations > MAX_LOOP_ITERATIONS + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "loop task `{}` maxIterations must be between 1 and {MAX_LOOP_ITERATIONS}", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}].loop.maxIterations")), + ); + continue; + } + let condition = loop_definition + .condition + .replace("loop.output", "vars.loopPrevious"); + if !is_exact_template(&condition) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::InvalidTemplate, + file, + format!( + "loop task `{}` while must be one exact typed condition", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}].loop.while")), + ); + continue; + } + if let Err(error) = validate_expression(&condition) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::InvalidTemplate, + file, + format!("loop task `{}`: {error}", task.id), + ) + .with_path(format!("spec.tasks[{position}].loop.while")), + ); + continue; + } + + let mut children: Vec = Vec::with_capacity(loop_definition.max_iterations); + for index in 0..loop_definition.max_iterations { + let mut bindings = JsonMap::new(); + bindings.insert("loopIndex".to_owned(), Value::from(index)); + bindings.insert( + "loopPrevious".to_owned(), + if let Some(previous) = children.last() { + Value::String(format!("${{{{ tasks.{previous}.output }}}}")) + } else { + loop_definition.initial.clone() + }, + ); + let id = expanded_task_id(&task.id, index, &bindings); + let mut child = task.clone(); + child.id.clone_from(&id); + child.loop_definition = None; + child.vars.extend(bindings.clone()); + child.when = Some(condition.clone()); + if let Some(previous) = children.last() { + child.needs.push(previous.clone()); + } + children.push(id.clone()); + expanded.push(ExpandedTaskDefinition { + definition: child, + source_position: position, + expansion: Some(CompiledExpansion { + parent: task.id.clone(), + index, + bindings, + }), + synthetic_use: None, + }); + } + + let mut aggregate = task.clone(); + aggregate.needs.clone_from(&children); + aggregate.foreach = None; + aggregate.matrix = None; + aggregate.route = None; + aggregate.loop_definition = None; + aggregate.memory_writes.clear(); + aggregate.when = None; + aggregate.vars.clear(); + aggregate.input.clear(); + aggregate.retry = RetryDefinition::default(); + aggregate.timeout_seconds = None; + aggregate.output_schema = None; + expanded.push(ExpandedTaskDefinition { + definition: aggregate, + source_position: position, + expansion: None, + synthetic_use: Some(TaskUse::LoopAggregate(CompiledLoopAggregate { + children, + condition, + })), + }); + continue; + } if task.route.is_some() && (task.foreach.is_some() || task.matrix.is_some()) { diagnostics.push( Diagnostic::error( @@ -202,7 +365,7 @@ fn expand_tasks( definition: task.clone(), source_position: position, expansion: None, - aggregate_children: None, + synthetic_use: None, }); continue; } @@ -382,7 +545,7 @@ fn expand_tasks( index, bindings: values, }), - aggregate_children: None, + synthetic_use: None, }); } @@ -402,7 +565,7 @@ fn expand_tasks( definition: aggregate, source_position: position, expansion: None, - aggregate_children: Some(children), + synthetic_use: Some(TaskUse::Aggregate(children)), }); } expanded @@ -414,6 +577,15 @@ fn expanded_task_id(parent: &str, index: usize, bindings: &JsonMap) -> String { format!("{parent}--{index:04}-{}", &digest[..12]) } +fn is_exact_template(value: &str) -> bool { + let trimmed = value.trim(); + trimmed.starts_with("${{") + && trimmed + .get(3..) + .and_then(|value| value.find("}}")) + .is_some_and(|closing| closing + 3 == trimmed.len() - 2) +} + pub fn compile(workflow: &Workflow, file: &str) -> Result> { let mut diagnostics = Vec::new(); let mut tasks = BTreeMap::new(); @@ -448,8 +620,8 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result JsonMap::new(), + TaskUse::Agent(_) + | TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) => JsonMap::new(), }; input.extend(task.input.clone()); - let memory_writes = if matches!(task_use, TaskUse::Aggregate(_) | TaskUse::Router(_)) { + let memory_writes = if matches!( + task_use, + TaskUse::Aggregate(_) | TaskUse::Router(_) | TaskUse::LoopAggregate(_) + ) { Vec::new() } else { task_memory_writes( @@ -539,7 +717,10 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + TaskUse::Action(_) + | TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) => JsonMap::new(), }; vars.extend(task.vars.clone()); declaration_order.push(task.id.clone()); @@ -630,6 +811,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result PlanPredictability::FullyPredictable, TaskUse::Router(_) => PlanPredictability::FullyPredictable, + TaskUse::LoopAggregate(_) => PlanPredictability::FullyPredictable, }; } let predictability = tasks @@ -724,7 +906,10 @@ fn task_memory_writes( .actions .get(name) .is_some_and(|action| action.kind == ActionKind::MemoryWrite), - TaskUse::Agent(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) => false, + TaskUse::Agent(_) + | TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) => false, }; if !is_memory_write { if !declared.is_empty() { @@ -805,13 +990,7 @@ fn validate_routers( .copied() .unwrap_or_default(); let route_path = format!("spec.tasks[{position}].route"); - let trimmed = router.select.trim(); - let exact_template = trimmed.starts_with("${{") - && trimmed - .get(3..) - .and_then(|value| value.find("}}")) - .is_some_and(|closing| closing + 3 == trimmed.len() - 2); - if !exact_template { + if !is_exact_template(&router.select) { diagnostics.push( Diagnostic::error( DiagnosticCode::InvalidTemplate, @@ -1093,7 +1272,7 @@ fn plan_requirements( predictability: task.predictability, }] } - TaskUse::Aggregate(_) | TaskUse::Router(_) => Vec::new(), + TaskUse::Aggregate(_) | TaskUse::Router(_) | TaskUse::LoopAggregate(_) => Vec::new(), }) .collect(); PlanRequirements { @@ -1840,6 +2019,170 @@ spec: })); } + #[test] + fn bounded_loop_expands_to_stable_sequential_iteration_tasks() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: bounded-loop } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: refine + uses: "action:assign" + loop: + maxIterations: 3 + while: "${{ vars.loopIndex < 2 }}" + initial: { value: seed } + with: + previous: "${{ vars.loopPrevious }}" + iteration: "${{ vars.loopIndex }}" + - { id: done, uses: "action:assign", needs: [refine] } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("compiles"); + let second = compile(&workflow, "fixture.yaml").expect("compiles deterministically"); + assert_eq!(plan, second); + + let loop_aggregate = match &plan.tasks["refine"].uses { + TaskUse::LoopAggregate(loop_aggregate) => loop_aggregate, + other => panic!("expected loop aggregate, got {other:?}"), + }; + assert_eq!(loop_aggregate.children.len(), 3); + assert_eq!(loop_aggregate.condition, "${{ vars.loopIndex < 2 }}"); + assert!(loop_aggregate.children[0].starts_with("refine--0000-")); + assert_eq!( + plan.tasks[&loop_aggregate.children[0]].vars["loopPrevious"], + serde_json::json!({"value": "seed"}) + ); + assert_eq!( + plan.tasks[&loop_aggregate.children[1]].vars["loopPrevious"], + Value::String(format!( + "${{{{ tasks.{}.output }}}}", + loop_aggregate.children[0] + )) + ); + assert_eq!( + plan.tasks[&loop_aggregate.children[1]].needs, + [loop_aggregate.children[0].clone()] + ); + assert_eq!( + plan.tasks[&loop_aggregate.children[2]].needs, + [loop_aggregate.children[1].clone()] + ); + assert_eq!(plan.tasks["refine"].needs, loop_aggregate.children); + assert_eq!(plan.tasks["done"].needs, ["refine"]); + } + + #[test] + fn bounded_loop_rejects_ambiguous_or_invalid_declarations() { + let invalid = [ + ( + r#"loop: { maxIterations: 2, while: "prefix ${{ vars.loopIndex < 1 }}" }"#, + "while must be one exact typed condition", + ), + ( + r#"loop: { maxIterations: 2, while: "${{ vars.loopIndex < \"two\" }}" }"#, + "unsupported expression", + ), + ]; + for (loop_definition, expected) in invalid { + let source = format!( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: {{ name: invalid-loop }} +spec: + actions: + assign: {{ kind: builtin.assign }} + tasks: + - id: refine + uses: "action:assign" + {loop_definition} +"# + ); + let workflow = parse(&source); + let diagnostics = compile(&workflow, "fixture.yaml").expect_err("loop rejected"); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains(expected)), + "{diagnostics:?}" + ); + } + + let collision = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: invalid-loop-collision } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: refine + uses: "action:assign" + vars: { loopIndex: 1 } + when: "${{ inputs.enabled }}" + foreach: { items: [a] } + loop: { maxIterations: 2, while: "${{ vars.loopIndex < 1 }}" } +"#, + ); + let diagnostics = compile(&collision, "fixture.yaml").expect_err("loop rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("cannot also declare foreach, matrix, or route") + })); + + let existing_when = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: invalid-loop-when } +spec: + inputs: { enabled: true } + actions: + assign: { kind: builtin.assign } + tasks: + - id: refine + uses: "action:assign" + when: "${{ inputs.enabled }}" + loop: { maxIterations: 2, while: "${{ vars.loopIndex < 1 }}" } +"#, + ); + let diagnostics = compile(&existing_when, "fixture.yaml").expect_err("loop rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("uses loop.while as its iteration guard") + })); + + let binding_collision = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: invalid-loop-binding } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: refine + uses: "action:assign" + vars: { loopPrevious: existing } + loop: { maxIterations: 2, while: "${{ vars.loopIndex < 1 }}" } +"#, + ); + let diagnostics = compile(&binding_collision, "fixture.yaml").expect_err("loop rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("bindings conflict with task vars") + })); + } + #[test] fn typed_router_cases_compile_to_explicit_destination_guards() { let workflow = parse( diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index 708f106..6694764 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -426,6 +426,8 @@ pub struct TaskDefinition { pub matrix: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub route: Option, + #[serde(default, skip_serializing_if = "Option::is_none", rename = "loop")] + pub loop_definition: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub memory_writes: Vec, #[serde(default)] @@ -489,6 +491,18 @@ pub struct RouteCaseDefinition { pub tasks: Vec, } +pub const MAX_LOOP_ITERATIONS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct LoopDefinition { + pub max_iterations: usize, + #[serde(rename = "while")] + pub condition: String, + #[serde(default)] + pub initial: Value, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RetryDefinition { @@ -1005,6 +1019,19 @@ fn validate_document(workflow: &Workflow, file: &str) -> Vec { .with_path(format!("spec.tasks[{position}].matrix.maxItems")), ); } + if let Some(loop_definition) = &task.loop_definition + && (loop_definition.max_iterations == 0 + || loop_definition.max_iterations > MAX_LOOP_ITERATIONS) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("loop.maxIterations must be between 1 and {MAX_LOOP_ITERATIONS}"), + ) + .with_path(format!("spec.tasks[{position}].loop.maxIterations")), + ); + } } for (name, action) in &workflow.spec.actions { if let Err(message) = action.validate_process_bounds() { @@ -1352,4 +1379,32 @@ spec: let diagnostics = parse_workflow(&source, "bad.yaml").expect_err("invalid bound"); assert!(diagnostics[0].message.contains("between 1 and 16777216")); } + + #[test] + fn rejects_loop_iteration_bounds_outside_framework_limits() { + for max_iterations in [0, MAX_LOOP_ITERATIONS + 1] { + let source = format!( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: {{ name: invalid-loop-bound }} +spec: + actions: + assign: {{ kind: builtin.assign }} + tasks: + - id: bounded + uses: action:assign + loop: + maxIterations: {max_iterations} + while: "${{{{ vars.loopIndex < 1 }}}}" +"# + ); + let diagnostics = parse_workflow(&source, "bad.yaml").expect_err("invalid loop bound"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("loop.maxIterations must be between 1 and 64") + })); + } + } } diff --git a/crates/agentctl-core/src/template.rs b/crates/agentctl-core/src/template.rs index 6be7a39..5692106 100644 --- a/crates/agentctl-core/src/template.rs +++ b/crates/agentctl-core/src/template.rs @@ -42,7 +42,9 @@ pub fn referenced_tasks(template: &str) -> BTreeSet { .trim() .strip_prefix("not ") .unwrap_or(expression.trim()); - let path = expression.split("==").next().unwrap_or(expression).trim(); + let path = split_comparison(expression) + .map_or(expression, |(left, _, _)| left) + .trim(); let mut parts = path.split('.'); (parts.next() == Some("tasks")) .then(|| parts.next().map(ToOwned::to_owned)) @@ -77,11 +79,30 @@ pub fn evaluate_when(expression: &str, context: &EvalContext) -> Result left_value == &right_value, + "!=" => left_value != &right_value, + "<" | "<=" | ">" | ">=" => { + let left_number = left_value + .as_f64() + .ok_or_else(|| TemplateError::Unsupported(candidate.to_owned()))?; + let right_number = right_value + .as_f64() + .ok_or_else(|| TemplateError::Unsupported(candidate.to_owned()))?; + match operator { + "<" => left_number < right_number, + "<=" => left_number <= right_number, + ">" => left_number > right_number, + ">=" => left_number >= right_number, + _ => unreachable!(), + } + } + _ => unreachable!(), + } } else { truthy(resolve_path(candidate, context)?) }; @@ -94,7 +115,7 @@ fn render_string(text: &str, context: &EvalContext) -> Result Result<(), TemplateError> { .trim() .strip_prefix("not ") .unwrap_or(expression.trim()); - let path = candidate - .split_once("==") - .map_or(candidate, |(left, _)| left) - .trim(); + let comparison = split_comparison(candidate); + let path = comparison.map_or(candidate, |(left, _, _)| left).trim(); + if let Some((_, operator, right)) = comparison { + if right.trim().is_empty() { + return Err(TemplateError::Unsupported(expression.to_owned())); + } + if matches!(operator, "<" | "<=" | ">" | ">=") + && serde_json::from_str::(right.trim()) + .ok() + .and_then(|value| value.as_f64()) + .is_none() + { + return Err(TemplateError::Unsupported(expression.to_owned())); + } + } let mut parts = path.split('.'); match parts.next() { Some("inputs" | "vars" | "memory") if parts.next().is_some() => {} @@ -174,6 +206,15 @@ fn validate_path_or_comparison(expression: &str) -> Result<(), TemplateError> { Ok(()) } +fn split_comparison(expression: &str) -> Option<(&str, &str, &str)> { + for operator in ["==", "!=", "<=", ">=", "<", ">"] { + if let Some((left, right)) = expression.split_once(operator) { + return Some((left, operator, right)); + } + } + None +} + fn resolve_path<'a>(path: &str, context: &'a EvalContext) -> Result<&'a Value, TemplateError> { validate_path_or_comparison(path)?; let parts: Vec<&str> = path.trim().split('.').collect(); @@ -258,20 +299,30 @@ mod tests { } #[test] - fn condition_supports_safe_equality_only() { - let inputs = BTreeMap::from([("deploy".to_owned(), Value::Bool(true))]); + fn condition_supports_typed_comparisons() { + let inputs = BTreeMap::from([ + ("deploy".to_owned(), Value::Bool(true)), + ("iteration".to_owned(), Value::from(2)), + ("label".to_owned(), Value::String("ready".to_owned())), + ]); let context = EvalContext { inputs, ..EvalContext::default() }; assert!(evaluate_when("${{ inputs.deploy == true }}", &context).expect("valid")); + assert!(evaluate_when("${{ inputs.label != \"blocked\" }}", &context).expect("valid")); + assert!(evaluate_when("${{ inputs.iteration < 3 }}", &context).expect("valid")); + assert!(evaluate_when("${{ inputs.iteration <= 2 }}", &context).expect("valid")); + assert!(evaluate_when("${{ inputs.iteration >= 2 }}", &context).expect("valid")); + assert!(!evaluate_when("${{ inputs.iteration > 2 }}", &context).expect("valid")); assert_eq!( render( - &Value::String("${{ inputs.deploy == true }}".to_owned()), + &Value::String("${{ inputs.iteration < 3 }}".to_owned()), &context ), Ok(Value::Bool(true)) ); + assert!(validate_expression("${{ inputs.label < \"z\" }}").is_err()); assert!(validate_expression("${{ inputs.x + 1 }}").is_err()); } diff --git a/crates/agentctl-core/tests/compatibility.rs b/crates/agentctl-core/tests/compatibility.rs index d718479..6278106 100644 --- a/crates/agentctl-core/tests/compatibility.rs +++ b/crates/agentctl-core/tests/compatibility.rs @@ -39,7 +39,10 @@ fn typescript_assign_fixture_translates_to_the_language_neutral_contract() { assert_eq!(expected.task.use_kind, "action"); assert_eq!(reference, &expected.task.reference); } - TaskUse::Agent(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) => { + TaskUse::Agent(_) + | TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) => { panic!("expected action task") } } diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index 2777f9b..a856850 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -2462,7 +2462,7 @@ impl Runtime { { continue; } - if !matches!(task.uses, TaskUse::Aggregate(_)) + if !matches!(task.uses, TaskUse::Aggregate(_) | TaskUse::LoopAggregate(_)) && dependencies.iter().any(|dependency| { matches!( dependency.state, @@ -3002,7 +3002,7 @@ impl Runtime { .iter() .filter_map(|needed| tasks.iter().find(|candidate| &candidate.task_id == needed)) .collect(); - if !matches!(task.uses, TaskUse::Aggregate(_)) + if !matches!(task.uses, TaskUse::Aggregate(_) | TaskUse::LoopAggregate(_)) && dependencies.iter().any(|dependency| { matches!( dependency.state, @@ -3457,6 +3457,65 @@ impl Runtime { memory: None, }) } + TaskUse::LoopAggregate(loop_aggregate) => { + let records = self.store.list_tasks(&run.run_id)?; + let items = loop_aggregate + .children + .iter() + .enumerate() + .map(|(index, child)| { + let record = records + .iter() + .find(|candidate| &candidate.task_id == child) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "loop iteration task `{child}` disappeared" + )) + })?; + Ok(serde_json::json!({ + "index": index, + "taskId": child, + "state": record.state, + "output": record.output, + "error": record.error, + })) + }) + .collect::, RuntimeError>>()?; + if let Some(last_id) = loop_aggregate.children.last() + && let Some(last) = records.iter().find(|record| &record.task_id == last_id) + && last.state == TaskState::Succeeded + { + let mut next_context = context.clone(); + next_context.vars.insert( + "loopIndex".to_owned(), + Value::from(loop_aggregate.children.len()), + ); + next_context.vars.insert( + "loopPrevious".to_owned(), + last.output.clone().unwrap_or(Value::Null), + ); + if evaluate_when(&loop_aggregate.condition, &next_context)? { + return Err(RuntimeError::Task { + task: task.id.clone(), + message: format!( + "loop reached maxIterations {} while its condition remained true", + loop_aggregate.children.len() + ), + }); + } + } + let iterations = items + .iter() + .filter(|item| item["state"] != "skipped") + .count(); + Ok(TaskExecution::Complete { + output: serde_json::json!({ + "iterations": iterations, + "items": items, + }), + memory: None, + }) + } } } @@ -4941,6 +5000,15 @@ fn task_output_schema(workflow: &Workflow, task: &agentctl_core::CompiledTask) - }, "additionalProperties": false })), + TaskUse::LoopAggregate(_) => Some(serde_json::json!({ + "type": "object", + "required": ["iterations", "items"], + "properties": { + "iterations": {"type": "integer", "minimum": 0}, + "items": {"type": "array"} + }, + "additionalProperties": false + })), }) } @@ -5035,6 +5103,11 @@ fn task_definition_fingerprint( "task": task, "router": router, }), + TaskUse::LoopAggregate(loop_aggregate) => serde_json::json!({ + "kind": "loop_aggregate", + "task": task, + "loop": loop_aggregate, + }), }; versioned_json_digest(&serde_json::json!({ "formatVersion": 1, @@ -9777,6 +9850,366 @@ spec: ); } + #[tokio::test] + async fn bounded_loop_records_iterations_and_supports_retry_repair_replay_and_cancellation() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let source = r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: bounded-loop } +spec: + policy: { approval: never } + actions: + assign: { kind: builtin.assign } + tasks: + - id: refine + uses: "action:assign" + loop: + maxIterations: 3 + while: "${{ vars.loopIndex < 2 }}" + initial: { value: seed } + with: + previous: "${{ vars.loopPrevious }}" + iteration: "${{ vars.loopIndex }}" +"#; + let (workflow, plan) = compile_fixture(source); + let children = match &plan.tasks["refine"].uses { + TaskUse::LoopAggregate(loop_aggregate) => loop_aggregate.children.clone(), + other => panic!("expected loop aggregate, got {other:?}"), + }; + let source_outcome = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("bounded loop succeeds"); + assert_eq!(source_outcome.state, RunState::Succeeded); + let source_tasks = store + .list_tasks(&source_outcome.run_id) + .expect("source tasks"); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == children[0]) + .expect("first iteration") + .state, + TaskState::Succeeded + ); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == children[1]) + .expect("second iteration") + .state, + TaskState::Succeeded + ); + assert_eq!( + source_tasks + .iter() + .find(|task| task.task_id == children[2]) + .expect("guarded iteration") + .state, + TaskState::Skipped + ); + let aggregate = source_tasks + .iter() + .find(|task| task.task_id == "refine") + .and_then(|task| task.output.as_ref()) + .expect("loop aggregate"); + assert_eq!(aggregate["iterations"], 2); + assert_eq!(aggregate["items"][2]["state"], "skipped"); + assert_eq!( + aggregate["items"][1]["output"]["output"]["iteration"], + serde_json::json!(1) + ); + + let retry_plan = runtime + .plan_retry( + &source_outcome.run_id, + &workflow, + &plan, + &[children[1].clone()], + false, + true, + ) + .expect("loop retry plan"); + assert!(retry_plan.compatible, "{:?}", retry_plan.blocked_reuse); + assert_eq!(retry_plan.reused_tasks, [children[0].clone()]); + assert!(retry_plan.rerun_tasks.contains(&children[1])); + assert!(retry_plan.rerun_tasks.contains(&children[2])); + assert!(retry_plan.rerun_tasks.contains(&"refine".to_owned())); + let retried = runtime + .retry( + &workflow, + &plan, + retry_plan, + Some("retry from iteration boundary"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("loop retry"); + assert_eq!(retried.state, RunState::Succeeded); + + let repair_plan = runtime + .plan_repair( + &source_outcome.run_id, + &workflow, + &plan, + &[children[1].clone()], + true, + ) + .expect("loop repair plan"); + assert!(repair_plan.compatible, "{:?}", repair_plan.blocked_reuse); + assert_eq!(repair_plan.reused_tasks, [children[0].clone()]); + let repaired = runtime + .repair( + &workflow, + &plan, + repair_plan, + Some("repair from iteration boundary"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("loop repair"); + assert_eq!(repaired.state, RunState::Succeeded); + let replay = runtime + .replay(&repaired.run_id) + .await + .expect("offline loop replay"); + assert_eq!(replay.state, RunState::Succeeded); + assert_eq!(replay.output, repaired.output); + assert!( + store + .list_effects(&replay.run_id) + .expect("replay effects") + .is_empty() + ); + + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let cancelled = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &cancellation, + ) + .await + .expect("loop cancellation"); + assert_eq!(cancelled.state, RunState::Cancelled); + assert!( + store + .list_tasks(&cancelled.run_id) + .expect("cancelled tasks") + .iter() + .all(|task| task.state == TaskState::Cancelled) + ); + } + + #[tokio::test] + async fn bounded_loop_handles_zero_iterations_and_fails_closed_at_its_bound() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (zero_workflow, zero_plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: zero-loop } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: bounded + uses: "action:assign" + loop: + maxIterations: 2 + while: "${{ vars.loopIndex < 0 }}" +"#, + ); + let zero = runtime + .start( + &zero_workflow, + &zero_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("zero-iteration loop succeeds"); + assert_eq!(zero.state, RunState::Succeeded); + let zero_tasks = store.list_tasks(&zero.run_id).expect("zero tasks"); + assert_eq!( + zero_tasks + .iter() + .find(|task| task.task_id == "bounded") + .and_then(|task| task.output.as_ref()) + .expect("zero aggregate")["iterations"], + 0 + ); + + let (one_workflow, one_plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: one-loop } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: bounded + uses: "action:assign" + loop: + maxIterations: 2 + while: "${{ vars.loopIndex < 1 }}" +"#, + ); + let one = runtime + .start( + &one_workflow, + &one_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("one-iteration loop succeeds"); + assert_eq!( + store + .list_tasks(&one.run_id) + .expect("one tasks") + .iter() + .find(|task| task.task_id == "bounded") + .and_then(|task| task.output.as_ref()) + .expect("one aggregate")["iterations"], + 1 + ); + + let (bound_workflow, bound_plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: exhausted-loop } +spec: + actions: + assign: { kind: builtin.assign } + tasks: + - id: bounded + uses: "action:assign" + loop: + maxIterations: 2 + while: "${{ vars.loopIndex < 3 }}" +"#, + ); + let failed_run_id = match runtime + .start( + &bound_workflow, + &bound_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { + run_id, + task, + message, + .. + }) => { + assert_eq!(task, "bounded"); + assert!(message.contains("reached maxIterations 2")); + run_id + } + other => panic!("expected exhausted loop failure, got {other:?}"), + }; + assert_eq!( + store + .list_tasks(&failed_run_id) + .expect("failed tasks") + .iter() + .find(|task| task.task_id == "bounded") + .expect("failed aggregate") + .state, + TaskState::Failed + ); + } + + #[tokio::test] + async fn bounded_loop_cancellation_keeps_in_flight_provider_effect_uncertain() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let provider = Arc::new(CancellationProvider::default()); + let runtime = runtime(store.clone(), directory.path()) + .with_registry(RuntimeRegistry::default().with_provider("fake", provider.clone())); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: loop-effect-cancellation } +spec: + policy: { approval: never } + providers: { fake: { kind: fake } } + agents: + worker: + provider: fake + model: fake + instructions: wait + maxTurns: 1 + tasks: + - id: bounded + uses: "agent:worker" + loop: + maxIterations: 2 + while: "${{ vars.loopIndex < 2 }}" + with: + prompt: "iteration ${{ vars.loopIndex }}" +"#, + ); + let cancellation = CancellationToken::new(); + let run_cancellation = cancellation.clone(); + let handle = tokio::spawn(async move { + runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &run_cancellation, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), async { + while provider.started.load(Ordering::SeqCst) != 1 { + provider.notify.notified().await; + } + }) + .await + .expect("first iteration started"); + cancellation.cancel(); + let outcome = handle.await.expect("join").expect("cancel outcome"); + assert_eq!(outcome.state, RunState::Cancelled); + let effects = store.list_effects(&outcome.run_id).expect("effects"); + assert_eq!(effects.len(), 1); + assert_eq!(effects[0].status, EffectStatus::Uncertain); + assert!( + store + .list_tasks(&outcome.run_id) + .expect("tasks") + .iter() + .all(|task| task.state == TaskState::Cancelled) + ); + } + #[tokio::test] async fn task_output_contract_failure_is_durable() { let directory = tempdir().expect("tempdir"); diff --git a/crates/agentctl-runtime/src/process.rs b/crates/agentctl-runtime/src/process.rs index 92c5ff0..9191b4f 100644 --- a/crates/agentctl-runtime/src/process.rs +++ b/crates/agentctl-runtime/src/process.rs @@ -395,7 +395,7 @@ mod tests { let error = run_bounded_process( shell(&script), limits(64, 64, 128), - Duration::from_secs(2), + Duration::from_secs(5), &CancellationToken::new(), ) .await diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 529b21f..69be5ea 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,9 +36,12 @@ Static foreach lists and matrix axes compile into ordinary namespaced child tasks followed by a pure aggregate. Their IDs, bindings, attempts, outputs, and recovery lineage use the same durable task model as authored nodes. Typed routers are pure tasks whose enumerated destination guards compile into the -graph; condition and route decisions are durable and replayable. Loops, -sub-workflows, handlers, compensation execution, and event triggers still -require their own explicit state and recovery contracts. The DSL carries +graph; condition and route decisions are durable and replayable. Bounded loops +compile into sequential namespaced iteration tasks and a pure aggregate, so +iteration attempts, effects, guard decisions, retry, repair, and replay use the +ordinary durable task model. Sub-workflows, handlers, compensation execution, +and event triggers still require their own explicit state and recovery +contracts. The DSL carries optional compensation metadata on a tool contract, but the runtime does not execute compensation. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index b25dbe8..4b76a4d 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -2,7 +2,7 @@ ## Preserved -Declaration-order scheduling among ready tasks, `needs` dataflow, exact typed templates, deterministic assign/assert/file/memory use cases, bounded agent/tool turns, approval concepts, SQLite local persistence, and the useful top-level command names remain. The language-neutral fixture records the legacy assign workflow’s translated model, graph order, and task reference. Omitted `foreach` and `matrix` fields preserve the unchanged single-task graph; compiled expansion metadata is additive. +Declaration-order scheduling among ready tasks, `needs` dataflow, exact typed templates, deterministic assign/assert/file/memory use cases, bounded agent/tool turns, approval concepts, SQLite local persistence, and the useful top-level command names remain. The language-neutral fixture records the legacy assign workflow’s translated model, graph order, and task reference. Omitted `foreach`, `matrix`, and `loop` fields preserve the unchanged single-task graph; compiled expansion metadata is additive. ## Migrated @@ -20,6 +20,6 @@ Unversioned YAML is compatibility-only and warns. The TypeScript package exposes Legacy workflows depending on packs, broad built-in tool profiles, remote MCP/A2A shape, MongoDB memory, provider-specific endpoint fields, or embedded credentials require manual conversion. The translator intentionally refuses to guess security-sensitive intent. -## Deferred product decisions +## Separate product decisions -Loops, sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. +Sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. Bounded loops are additive; unbounded or model-controlled iteration is intentionally unsupported. diff --git a/docs/DSL.md b/docs/DSL.md index c161c55..1a3b026 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,7 +2,7 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach` or `matrix` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach`, `matrix`, or `loop` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. @@ -30,8 +30,15 @@ tasks plus a parent aggregate. `maxItems` defaults to 32, expansion cannot exceed 256 children, and model output cannot drive it. Retry and repair can select the visible child IDs. See [Matrix and foreach tasks](guides/MATRIX_AND_FOREACH.md). +Bounded `loop` tasks require `maxIterations` from 1 through 64 and one exact +typed `while` guard. They compile into stable sequential iteration tasks. +`vars.loopIndex` is the zero-based position and `vars.loopPrevious` is the +initial value or preceding iteration output. A still-true guard after the +maximum fails closed. Retry and repair select iteration IDs. See [Bounded +loops](guides/BOUNDED_LOOPS.md). + `builtin.shell.exec` captures stdout and stderr concurrently. Its optional `stdoutLimitBytes`, `stderrLimitBytes`, and `combinedOutputLimitBytes` fields default to 1 MiB, 1 MiB, and 2 MiB respectively. Each configured value must be between 1 byte and 16 MiB. `timeoutSeconds` must be between 1 and 86,400. Exceeding an output bound terminates and reaps the process and records a structured failed effect; timeout or cancellation remains an uncertain effect because external changes may already have occurred. These fields are validated identically for workflow and pack actions. The parser translates a limited unversioned `playbook:` document and emits a migration warning. Use `agentctl migrate old.yaml --write new.yaml`. Legacy pack-backed, MCP, A2A, provider-specific, and broad module configurations need manual migration; see [Migrating from TypeScript](MIGRATING_FROM_TYPESCRIPT.md). -Not implemented in v1alpha1: loops, sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. +Not implemented in v1alpha1: sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index 52a94b2..f3f7e65 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -43,4 +43,9 @@ Pure router tasks retain their typed selected value and enumerated destination IDs. Skipped branch records are copied directly by recorded replay and never pass through a running state. +Bounded loops compile into a fixed sequential chain of ordinary tasks plus a +pure aggregate. Each iteration retains its guard decision, output, effects, +artifacts, attempts, and recovery identity. A false guard skips the remaining +chain. A guard that remains true after the declared maximum fails closed. + The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 18ba122..103d23f 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -23,7 +23,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an These are useful extensions but are not required by the product thesis. They need new deterministic state and compatibility contracts before implementation: -- loops; sub-workflows; compensation execution; +- sub-workflows; compensation execution; - structured agent teams and handoffs; - model token streaming into CLI/workflow state; - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; @@ -46,9 +46,13 @@ These are useful extensions but are not required by the product thesis. They nee - Foreach and matrix expansion accepts only static workflow values, requires `maxItems`, and is capped at 256 children. Runtime or model-controlled graph growth is not supported. -- Conditions support typed paths, equality, and `not`; routers support exact - typed selectors and enumerated destinations. Arbitrary expressions, implicit - dependencies, and model-owned hidden routing are rejected. +- Conditions support typed paths, equality, inequality, numeric ordering, and + `not`; routers support exact typed selectors and enumerated destinations. + Arbitrary expressions, implicit dependencies, and model-owned hidden routing + are rejected. +- Loops are sequential, require a maximum from 1 through 64, and compile all + iteration boundaries before execution. Runtime or model-controlled graph + growth and unbounded loops are rejected. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. diff --git a/docs/adr/0005-narrow-v1-scheduling-and-extensions.md b/docs/adr/0005-narrow-v1-scheduling-and-extensions.md index 356be54..9faf463 100644 --- a/docs/adr/0005-narrow-v1-scheduling-and-extensions.md +++ b/docs/adr/0005-narrow-v1-scheduling-and-extensions.md @@ -1,7 +1,12 @@ # ADR 0005: Narrow v1 scheduling and extension surface -Status: superseded for scheduling by ADR 0008, 2026-07-24. +Status: superseded for scheduling and control flow by ADRs 0008 through 0011, +2026-07-24. V1alpha1 schedules a sequential DAG in declaration order and integrates typed actions/tools, local packs, MCP 2025-11-25, and A2A 1.0. `maxConcurrency` greater than one is rejected. -Parallel groups, loops, routing, sub-workflows, teams/handoffs, automatic reconnection/resubmission, executable plugin ABIs, and registries are deferred. Each needs deterministic merge, cancellation, policy, version, and recovery semantics before it can enter the stable contract. +The original decision excluded parallel execution, loops, and routing until +their deterministic semantics existed. ADRs 0008, 0010, and 0011 now define +those contracts. Sub-workflows, teams/handoffs, automatic +reconnection/resubmission, executable plugin ABIs, and registries remain +separate product decisions. diff --git a/docs/adr/0010-typed-routing-and-durable-decisions.md b/docs/adr/0010-typed-routing-and-durable-decisions.md index 9bc3e83..3b44aca 100644 --- a/docs/adr/0010-typed-routing-and-durable-decisions.md +++ b/docs/adr/0010-typed-routing-and-durable-decisions.md @@ -4,7 +4,7 @@ Status: accepted ## Decision -Conditions remain a constrained path, equality, and `not` language. Router +Conditions remain a constrained path, typed comparison, and `not` language. Router tasks are pure compiled nodes with one exact typed selector, unique JSON case values, enumerated destination tasks, and optional default destinations. Every destination declares the router as a dependency. diff --git a/docs/adr/0011-bounded-loops-as-static-graphs.md b/docs/adr/0011-bounded-loops-as-static-graphs.md new file mode 100644 index 0000000..b6d32c7 --- /dev/null +++ b/docs/adr/0011-bounded-loops-as-static-graphs.md @@ -0,0 +1,25 @@ +# ADR 0011: bounded loops as static graphs + +Status: accepted + +## Decision + +A task-level loop compiles into a statically bounded chain of iteration tasks +and one pure aggregate. `maxIterations` is mandatory and capped at 64. The +typed `while` condition is evaluated before each iteration. A false condition +durably skips the remaining chain. A still-true condition after the final +iteration fails the aggregate. + +Each iteration has a stable digest-qualified ID, a zero-based `loopIndex`, and +a typed `loopPrevious` binding containing either the declared initial value or +the preceding iteration's full output. + +## Consequences + +- The scheduler never accepts model-controlled graph growth. +- Existing task attempts, effects, approvals, artifacts, retry, repair, + cancellation, and replay semantics apply at every iteration boundary. +- The compiled graph contains at most 64 iteration nodes per loop declaration. +- Loops are sequential. Independent graph branches remain the mechanism for + deterministic parallel work. +- Authors cannot combine `loop` with another task expansion, router, or `when`. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index e8eaba6..77fc4c1 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -75,9 +75,10 @@ cargo xtask acceptance-container | Sensitive-state encryption | authenticated context, wrong-key, tamper, inventory, stale-writer trigger, rollback, rotation, checkpoint, and retained-schema tests passed | packaged CLI scenario 31 and the 12-stage verification gate passed | verified | | Secret references | environment compatibility, file bounds/missing/symlink containment, process allowlist/timeout/output/cancellation, zeroizing values, adapter redaction, and raw-database absence tests passed | packaged CLI scenario 32 and the 12-stage verification gate passed | verified | | Parallel scheduling | overlap, caps, conflicts, ordered atomic commits, approvals, cancellation, retry, repair, and replay tests passed | packaged CLI scenario 33 and OCI parallel run/replay passed | deterministic verified; live pending | -| Foreach/matrix | compiler bounds/identity tests and runtime partial-failure, child retry, sibling reuse, aggregation, and replay tests passed | packaged CLI scenario 34 pending | deterministic in progress | -| Conditions/routers | compiler typed-case/guard failures and runtime durable condition, route, retry, changed-input repair, and skipped replay tests passed | packaged CLI scenario 35 pending | deterministic in progress | -| Loops/sub-workflows | pending | pending | open | +| Foreach/matrix | compiler bounds/identity tests and runtime partial-failure, child retry, sibling reuse, aggregation, and replay tests passed | packaged CLI scenario 34 passed | deterministic verified; live pending | +| Conditions/routers | compiler typed-case/guard failures and runtime durable condition, route, retry, changed-input repair, and skipped replay tests passed | packaged CLI scenario 35 passed | deterministic verified; live pending | +| Bounded loops | compiler bounds/identity tests and runtime zero/one/max, exhaustion, cancellation, uncertain effect, retry, repair, and replay tests passed | packaged CLI scenario 36 passed | deterministic verified; live pending | +| Sub-workflows | pending | pending | open | | Compensation/handoffs/streaming | pending | pending | open | | MCP/A2A resilience | pending | pending | open | | Packs/trust/extensions | pending | pending | open | diff --git a/docs/execution/DECISIONS.md b/docs/execution/DECISIONS.md index af50fe3..5649499 100644 --- a/docs/execution/DECISIONS.md +++ b/docs/execution/DECISIONS.md @@ -12,5 +12,6 @@ | [0008](../adr/0008-deterministic-parallel-batches.md) | Deterministic parallel batches | accepted | Bounded overlap, isolated snapshots, declared writes, and atomic plan-order commits. | | [0009](../adr/0009-bounded-static-task-expansion.md) | Bounded static task expansion | accepted | Stable child tasks and ordered aggregates prevent model-controlled graph growth. | | [0010](../adr/0010-typed-routing-and-durable-decisions.md) | Typed routing and durable decisions | accepted | Pure enumerated routers and hashed condition contexts make branching inspectable and replayable. | +| [0011](../adr/0011-bounded-loops-as-static-graphs.md) | Bounded loops as static graphs | accepted | Fixed iteration chains reuse ordinary durable task and recovery semantics. | These decisions resolve the researched patterns in [LANDSCAPE.md](../research/LANDSCAPE.md). No unsafe code or distributed control plane ADR is required because neither exists. diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md index d71444c..e35f27a 100644 --- a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -38,6 +38,7 @@ This inventory is enforced by `cargo xtask examples-verify`. The default command | `examples/v1/google-live.yaml` | Google native provider | Google | credentialed execution | 0 | 0 | N/A | Protocol mock | External opt-in | N/A | N/A | Static | passed | | `examples/v1/hello.yaml` | Minimal assign workflow | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | | `examples/v1/long-term-memory.yaml` | Namespaced durable memory | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | +| `examples/v1/loop.yaml` | Bounded durable loop and ordered iteration aggregation | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Iteration states and values | passed | | `examples/v1/matrix.yaml` | Bounded static matrix and ordered aggregation | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Aggregate states and values | passed | | `examples/v1/mcp.yaml` | MCP call contract | MCP | external execution | 0 | 0 | Protocol mock | Protocol mock | N/A | N/A | N/A | Static | passed | | `examples/v1/openai-live.yaml` | Minimal OpenAI response | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Live gate | live passed | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index 3c7bf1a..8bdb023 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -46,7 +46,7 @@ complete, every entry must have exactly one final disposition: | SCH-001 | Deterministic parallel execution | in progress | implemented | | DYN-001 | Foreach and matrix | in progress | implemented | | COND-001 | Conditions and routers | in progress | implemented | -| LOOP-001 | Bounded loops | open | implemented | +| LOOP-001 | Bounded loops | in progress | implemented | | SUB-001 | Sub-workflows | open | implemented | | COMP-001 | Compensation | open | implemented | | TEAM-001 | Structured teams and handoffs | open | redesigned | @@ -288,7 +288,8 @@ complete, every entry must have exactly one final disposition: ### LOOP-001: Bounded loops -- Current behavior: loops are rejected. +- Current behavior: a task-level loop compiles into a bounded sequential chain + of durable iteration tasks and one pure aggregate. - User impact: bounded refine/verify workflows require duplicated tasks. - Security or durability impact: an unbounded model-owned loop violates the runtime's bounded-execution thesis. @@ -297,13 +298,23 @@ complete, every entry must have exactly one final disposition: - Required implementation: stable iteration IDs, durable iteration state, outputs, cancellation, effect identities, repair/retry at boundaries, replay, and loop/resource budgets. -- Migration impact: plan, checkpoint, and task-attempt formats. +- Migration impact: the DSL and compiled plan gain additive loop records. + Iterations use existing task, checkpoint, effect, and attempt storage, so no + SQLite migration is required. - Tests: zero/one/max iterations, bound exceeded, cancellation, uncertain effect, repair/retry, and replay. - Examples: bounded operational verification loop. - Live evidence: a two-iteration maximum agent scenario. - Documentation: loop safety and recovery. -- Final disposition: pending implementation evidence. +- Final disposition: implemented. Iteration IDs and bindings are stable, + `maxIterations` is required and capped at 64, typed guards run before each + iteration, false guards durably skip the remaining chain, and a still-true + final guard fails closed. Deterministic verification covers zero, one, and + maximum iterations, bound exhaustion, cancellation with an uncertain + in-flight provider effect, per-boundary retry and repair, offline replay, and + zero replay effects. Packaged CLI scenario 36 verifies plan, run, inspect, + and replay. Program state remains in progress until the bounded live agent + scenario executes. ### SUB-001: Reusable sub-workflows diff --git a/docs/guides/BOUNDED_LOOPS.md b/docs/guides/BOUNDED_LOOPS.md new file mode 100644 index 0000000..7ced33c --- /dev/null +++ b/docs/guides/BOUNDED_LOOPS.md @@ -0,0 +1,79 @@ +# Bounded loops + +A task-level `loop` compiles into a fixed sequence of ordinary durable +iteration tasks and one pure aggregate. The graph exists before execution, so +neither a model nor runtime data can create an unbounded number of tasks. + +```yaml +tasks: + - id: refine + uses: agent:reviewer + loop: + maxIterations: 3 + while: "${{ vars.loopIndex < 2 }}" + initial: + status: new + with: + prompt: "Iteration ${{ vars.loopIndex }}" + previous: "${{ vars.loopPrevious }}" +``` + +`maxIterations` is required and accepts 1 through 64. `while` is an exact +typed condition. It is evaluated before each iteration. Numeric ordering +supports `<`, `<=`, `>`, and `>=`; equality and inequality use `==` and `!=`. +Ordering compares numbers only. + +The body receives: + +- `vars.loopIndex`: the zero-based iteration number. +- `vars.loopPrevious`: `initial` for iteration zero, then the complete output + of the preceding iteration. + +The condition may use `loop.output` as an authoring alias for +`vars.loopPrevious`, including nested paths. A task cannot combine `loop` with +`when`, `foreach`, `matrix`, or `route`, and loop bindings cannot shadow +task-local variables. + +## Durable identity and output + +Iteration IDs have the same stable +`PARENT--INDEX-BINDING_DIGEST` form as static expansion. Each iteration keeps +its own attempts, condition decision, output, error, effects, artifacts, and +audit history. `agentctl plan` exposes every ID and binding. + +When the guard becomes false, that iteration and all remaining precompiled +iterations become `skipped`. The parent aggregate still succeeds and returns: + +```json +{ + "iterations": 2, + "items": [ + { + "index": 0, + "taskId": "refine--0000-...", + "state": "succeeded", + "output": {}, + "error": null + } + ] +} +``` + +`iterations` counts attempted, non-skipped iterations. If the condition remains +true after the final allowed iteration, the aggregate fails closed with a +maximum-iteration error. + +## Recovery and effects + +Retry and repair select iteration IDs, so a compatible prefix can be reused +while the selected boundary and its descendants execute again. Reused +iterations dispatch no providers, tools, processes, or network calls. Fresh +iterations use the normal effect ledger, idempotency, approval, and uncertain +effect reconciliation rules. Recorded replay copies iteration and aggregate +results without executing effects. + +Cancellation marks every unfinished iteration and the aggregate cancelled. +The loop body's `failure` and retry settings apply independently to each +iteration. + +See [`examples/v1/loop.yaml`](../../examples/v1/loop.yaml). diff --git a/docs/guides/CONDITIONS_AND_ROUTERS.md b/docs/guides/CONDITIONS_AND_ROUTERS.md index 6fb2760..aa440cd 100644 --- a/docs/guides/CONDITIONS_AND_ROUTERS.md +++ b/docs/guides/CONDITIONS_AND_ROUTERS.md @@ -5,7 +5,7 @@ constrained template evaluator and cannot execute code. ## Conditions -`when` accepts one path, optional `not`, or equality against a JSON value: +`when` accepts one path, optional `not`, or a typed comparison: ```yaml vars: @@ -31,6 +31,10 @@ sets the task to `skipped` with this durable output shape: For a true condition, the same decision is retained in the transition audit before the normal task output replaces the temporary decision value. +`==` and `!=` accept JSON literals or strings. `<`, `<=`, `>`, and `>=` accept +numeric literals and require a numeric path value. Arithmetic, functions, +indexing, and path-to-path comparisons are rejected. + ## Routers A router is a pure task with an exact typed selector and enumerated diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index b5cb100..9764b84 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -45,6 +45,7 @@ Each task requires `id` and `uses`. `uses` is `action:name`, `agent:name`, or | `foreach` | none | Static typed `items`, binding `as`, and `maxItems`. Mutually exclusive with `matrix`; maximum 256 children. | | `matrix` | none | Static `axes` Cartesian product and `maxItems`. Axis names are template-safe identifiers; maximum 256 children. | | `route` | required for `uses: router` | Exact typed `select`, unique typed cases, enumerated destinations, and optional default destinations. Every destination must depend on the router. | +| `loop` | none | Required `maxIterations` from 1 through 64, exact typed `while`, and optional typed `initial` value. Mutually exclusive with `when`, `foreach`, `matrix`, and `route`. | | `memoryWrites` | inferred or `[]` | Working-memory keys. Literal memory-write keys are inferred; templated keys require an explicit set. Unordered overlaps fail when concurrency is greater than one. | | `when` | true | Constrained boolean/equality expression. | | `vars` | `{}` | Task-local JSON values. | @@ -57,7 +58,8 @@ Each task requires `id` and `uses`. `uses` is `action:name`, `agent:name`, or Ready tasks are selected in YAML declaration order up to `maxConcurrency`. They read isolated durable snapshots and commit in compiled order. There is no runtime or model-controlled expansion. Static `foreach` and `matrix` tasks -compile to inspectable child tasks and a parent aggregate. There is no loop, +compile to inspectable child tasks and a parent aggregate. Bounded loops +compile to a sequential child chain and parent aggregate. There is no sub-workflow, handler, or separate parallel group in this version. ## Agents @@ -101,7 +103,7 @@ ${{ memory.path }} ${{ tasks.task-id.output.path }} ``` -An exact template preserves objects, arrays, booleans, numbers, strings, and null. Text interpolation accepts scalars. Conditions add `not` and equality against a JSON literal or string. There is no code execution, arithmetic, arbitrary function, indexing, or implicit dependency. +An exact template preserves objects, arrays, booleans, numbers, strings, and null. Text interpolation accepts scalars. Conditions add `not`, type-sensitive `==` and `!=`, and numeric `<`, `<=`, `>`, and `>=`. There is no code execution, arithmetic, arbitrary function, indexing, or implicit dependency. ## Secret references @@ -124,6 +126,7 @@ agentctl run examples/v1/dataflow.yaml --db /tmp/dataflow.db --output json --col Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Matrix and foreach](../guides/MATRIX_AND_FOREACH.md), [Conditions and -routers](../guides/CONDITIONS_AND_ROUTERS.md), [Secret +routers](../guides/CONDITIONS_AND_ROUTERS.md), [Bounded +loops](../guides/BOUNDED_LOOPS.md), [Secret references](../guides/SECRET_REFERENCES.md), [Policies](../policies.md), [Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). diff --git a/examples/v1/README.md b/examples/v1/README.md index 6b65f9d..7e186e2 100644 --- a/examples/v1/README.md +++ b/examples/v1/README.md @@ -12,6 +12,7 @@ The deterministic examples are exercised by `cargo xtask verify` and never requi - `crash-resume.yaml`: effect-ledger write followed by observation; crash behavior is injected in runtime tests. - `parallel.yaml`: bounded parallel batches with disjoint working-memory writes and stable commits. - `matrix.yaml`: bounded static matrix expansion, stable child identities, and ordered aggregation. +- `loop.yaml`: bounded sequential iteration, stable boundaries, aggregation, and fail-closed exhaustion. - `working-memory.yaml` and `long-term-memory.yaml`: separate memory lifecycles. - `fake-provider.yaml`: deterministic model-provider path. - `mcp.yaml` and `a2a.yaml`: local protocol fixtures, backed by the protocol crate's mock-server tests. diff --git a/examples/v1/loop.yaml b/examples/v1/loop.yaml new file mode 100644 index 0000000..9e8fd90 --- /dev/null +++ b/examples/v1/loop.yaml @@ -0,0 +1,24 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: bounded-loop +spec: + policy: + approval: never + outputs: + iterations: "${{ tasks.refine.output.iterations }}" + results: "${{ tasks.refine.output.items }}" + actions: + assign: + kind: builtin.assign + tasks: + - id: refine + uses: action:assign + loop: + maxIterations: 3 + while: "${{ vars.loopIndex < 2 }}" + initial: + value: seed + with: + previous: "${{ vars.loopPrevious }}" + iteration: "${{ vars.loopIndex }}" diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index 91ab1d1..5183f9c 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -732,6 +732,16 @@ } ] }, + "loop": { + "anyOf": [ + { + "$ref": "#/$defs/LoopDefinition" + }, + { + "type": "null" + } + ] + }, "memoryWrites": { "type": "array", "items": { @@ -871,6 +881,27 @@ "tasks" ] }, + "LoopDefinition": { + "type": "object", + "properties": { + "maxIterations": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "while": { + "type": "string" + }, + "initial": { + "default": null + } + }, + "additionalProperties": false, + "required": [ + "maxIterations", + "while" + ] + }, "FailureBehavior": { "type": "string", "enum": [ diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 2228af7..0674149 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 35; +const ACCEPTANCE_SCENARIOS: usize = 36; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1490,6 +1490,69 @@ pub fn run(root: &Path) -> Result<()> { let router_replay_inspect = inspect(&binary, root, &router_db, router_replay_id)?; ensure!(array_len(&router_replay_inspect, "/data/effects")? == 0); + scenario( + 36, + "packaged CLI runs, inspects, and replays bounded durable loop iterations", + ); + let loop_workflow = root.join("examples/v1/loop.yaml"); + let loop_plan = successful_json( + &binary, + root, + &strings([ + "plan", + path(&loop_workflow)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&loop_plan, "/data/tasks/refine/uses/kind", "loop_aggregate")?; + let loop_children = loop_plan + .pointer("/data/tasks/refine/uses/name/children") + .and_then(Value::as_array) + .context("loop iteration child list")?; + ensure!(loop_children.len() == 3); + let first_loop_child = loop_children[0] + .as_str() + .context("first loop iteration ID")?; + ensure!(first_loop_child.starts_with("refine--0000-")); + let loop_db = directory.path().join("loop.db"); + let loop_run = successful_json( + &binary, + root, + &run_args(&loop_workflow, &loop_db, root, &[]), + )?; + ensure_eq(&loop_run, "/data/state", "succeeded")?; + ensure_eq(&loop_run, "/data/output/iterations", 2_u64)?; + ensure_eq( + &loop_run, + "/data/output/results/1/output/output/iteration", + 1_u64, + )?; + ensure_eq(&loop_run, "/data/output/results/2/state", "skipped")?; + let loop_run_id = string_at(&loop_run, "/data/runId")?; + let loop_inspect = inspect(&binary, root, &loop_db, loop_run_id)?; + ensure!(array_len(&loop_inspect, "/data/tasks")? == 4); + let loop_replay = successful_json( + &binary, + root, + &strings([ + "replay", + loop_run_id, + "--db", + path(&loop_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&loop_replay, "/data/state", "succeeded")?; + let loop_replay_id = string_at(&loop_replay, "/data/runId")?; + let loop_replay_inspect = inspect(&binary, root, &loop_db, loop_replay_id)?; + ensure!(array_len(&loop_replay_inspect, "/data/effects")? == 0); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } From 6c4c92db429c8f956e77631f678b4ded890ac398 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Fri, 24 Jul 2026 23:35:57 +0530 Subject: [PATCH 15/44] feat: add reusable sub-workflows --- README.md | 1 + crates/agentctl-cli/src/main.rs | 25 + crates/agentctl-core/src/compiler.rs | 637 +++++++++++++++++- crates/agentctl-core/src/dsl.rs | 15 + crates/agentctl-core/src/pack.rs | 21 +- crates/agentctl-core/tests/compatibility.rs | 4 +- crates/agentctl-runtime/src/lib.rs | 307 ++++++++- docs/ARCHITECTURE.md | 8 +- docs/COMPATIBILITY.md | 2 +- docs/DSL.md | 10 +- docs/DURABLE_EXECUTION.md | 5 + docs/LIMITATIONS.md | 5 +- docs/PACKS.md | 6 +- .../0012-subworkflows-as-namespaced-graphs.md | 26 + docs/execution/COMPLETENESS_VERIFICATION.md | 2 +- docs/execution/DECISIONS.md | 1 + docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 21 +- docs/guides/SUB_WORKFLOWS.md | 72 ++ docs/reference/YAML.md | 13 +- examples/v1/README.md | 3 +- examples/v1/example.pack.yaml | 24 + examples/v1/reusable-pack.yaml | 9 +- examples/v1/subworkflow.yaml | 48 ++ schemas/workflow.schema.json | 40 ++ xtask/src/acceptance.rs | 66 +- 26 files changed, 1321 insertions(+), 51 deletions(-) create mode 100644 docs/adr/0012-subworkflows-as-namespaced-graphs.md create mode 100644 docs/guides/SUB_WORKFLOWS.md create mode 100644 examples/v1/subworkflow.yaml diff --git a/README.md b/README.md index 276f40e..656904d 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ For bounded independent branches and working-memory conflict rules, use [Determi For bounded static task expansion and child-level recovery, use [Matrix and foreach tasks](docs/guides/MATRIX_AND_FOREACH.md). For typed branching and durable decisions, use [Conditions and routers](docs/guides/CONDITIONS_AND_ROUTERS.md). For iterative work with a hard execution ceiling and iteration-level recovery, use [Bounded loops](docs/guides/BOUNDED_LOOPS.md). +For typed reusable graphs with namespaced recovery boundaries, use [Reusable sub-workflows](docs/guides/SUB_WORKFLOWS.md). ## Safety boundary diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index 130c0ef..07d56bf 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -2096,6 +2096,11 @@ fn load_packs(workflow: &mut Workflow, workflow_path: &Path) -> Result<(), CliEr for agent in pack.agents.values_mut() { agent.tools = agent.tools.iter().map(|name| qualify(name)).collect(); } + for definition in pack.workflows.values_mut() { + for task in &mut definition.tasks { + qualify_pack_task(task, &qualify); + } + } for (name, action) in pack.actions { insert_pack_item( &mut workflow.spec.actions, @@ -2110,10 +2115,30 @@ fn load_packs(workflow: &mut Workflow, workflow_path: &Path) -> Result<(), CliEr for (name, agent) in pack.agents { insert_pack_item(&mut workflow.spec.agents, qualify(&name), agent, &pack.name)?; } + for (name, definition) in pack.workflows { + insert_pack_item( + &mut workflow.spec.subworkflows, + qualify(&name), + definition, + &pack.name, + )?; + } } Ok(()) } +fn qualify_pack_task( + task: &mut agentctl_core::dsl::TaskDefinition, + qualify: &impl Fn(&str) -> String, +) { + for prefix in ["action:", "agent:", "workflow:"] { + if let Some(name) = task.uses.strip_prefix(prefix) { + task.uses = format!("{prefix}{}", qualify(name)); + break; + } + } +} + fn insert_pack_item( target: &mut BTreeMap, name: String, diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index 73fcdb5..ab9f3c7 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -78,6 +78,21 @@ pub struct CompiledLoopAggregate { pub condition: String, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledSubworkflowInput { + pub name: String, + pub version: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledSubworkflowAggregate { + pub name: String, + pub version: String, + pub children: Vec, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "kind", content = "name")] pub enum TaskUse { @@ -86,6 +101,8 @@ pub enum TaskUse { Aggregate(Vec), Router(CompiledRouter), LoopAggregate(CompiledLoopAggregate), + SubworkflowInput(CompiledSubworkflowInput), + SubworkflowAggregate(CompiledSubworkflowAggregate), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -190,9 +207,19 @@ fn expand_tasks( workflow: &Workflow, file: &str, diagnostics: &mut Vec, + synthetic_uses: &BTreeMap, ) -> Vec { let mut expanded = Vec::new(); for (position, task) in workflow.spec.tasks.iter().enumerate() { + if let Some(task_use) = synthetic_uses.get(&task.id) { + expanded.push(ExpandedTaskDefinition { + definition: task.clone(), + source_position: position, + expansion: None, + synthetic_use: Some(task_use.clone()), + }); + continue; + } if let Some(loop_definition) = &task.loop_definition { let loop_path = format!("spec.tasks[{position}].loop"); if task.foreach.is_some() || task.matrix.is_some() || task.route.is_some() { @@ -586,12 +613,389 @@ fn is_exact_template(value: &str) -> bool { .is_some_and(|closing| closing + 3 == trimmed.len() - 2) } +fn expand_subworkflow_calls( + workflow: &Workflow, + file: &str, + diagnostics: &mut Vec, +) -> (Workflow, BTreeMap) { + let mut flattened = workflow.clone(); + let mut tasks = Vec::new(); + let mut synthetic = BTreeMap::new(); + for task in &workflow.spec.tasks { + instantiate_subworkflow_task( + workflow, + task.clone(), + file, + &mut Vec::new(), + &mut tasks, + &mut synthetic, + diagnostics, + ); + } + flattened.spec.tasks = tasks; + (flattened, synthetic) +} + +#[allow(clippy::too_many_arguments)] +fn instantiate_subworkflow_task( + workflow: &Workflow, + task: TaskDefinition, + file: &str, + stack: &mut Vec, + tasks: &mut Vec, + synthetic: &mut BTreeMap, + diagnostics: &mut Vec, +) { + let Some(name) = task.uses.strip_prefix("workflow:") else { + tasks.push(task); + return; + }; + let Some(definition) = workflow.spec.subworkflows.get(name) else { + diagnostics.push(Diagnostic::error( + DiagnosticCode::MissingReference, + file, + format!( + "sub-workflow invocation `{}` refers to unknown workflow `{name}`", + task.id + ), + )); + return; + }; + if stack.iter().any(|active| active == name) { + let mut cycle = stack.clone(); + cycle.push(name.to_owned()); + diagnostics.push(Diagnostic::error( + DiagnosticCode::DependencyCycle, + file, + format!("sub-workflow cycle: {}", cycle.join(" -> ")), + )); + return; + } + if semver::Version::parse(&definition.version).is_err() { + diagnostics.push(Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("sub-workflow `{name}` version must be semantic versioning"), + )); + return; + } + for (label, schema) in [ + ("inputSchema", &definition.input_schema), + ("outputSchema", &definition.output_schema), + ] { + if let Err(error) = jsonschema::validator_for(schema) { + diagnostics.push(Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!("sub-workflow `{name}` {label} is not valid JSON Schema: {error}"), + )); + return; + } + } + if task.foreach.is_some() + || task.matrix.is_some() + || task.route.is_some() + || task.loop_definition.is_some() + || task.when.is_some() + || !task.vars.is_empty() + || !task.memory_writes.is_empty() + || task.retry != RetryDefinition::default() + || task.timeout_seconds.is_some() + || task.output_schema.is_some() + { + diagnostics.push(Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + format!( + "sub-workflow invocation `{}` accepts needs, with, and failure only", + task.id + ), + )); + return; + } + + let identity = sha256(format!("{name}@{}", definition.version).as_bytes()); + let input_id = format!("{}--inputs-{}", task.id, &identity[..8]); + let namespace = format!("{}--", task.id); + let local_ids = definition + .tasks + .iter() + .map(|child| (child.id.clone(), format!("{namespace}{}", child.id))) + .collect::>(); + if local_ids.values().any(|id| id == &input_id) { + diagnostics.push(Diagnostic::error( + DiagnosticCode::DuplicateTask, + file, + format!("sub-workflow `{name}` produces reserved task ID `{input_id}`"), + )); + return; + } + + let mut values = definition.inputs.clone(); + values.extend(task.input.clone()); + let mut input_boundary = synthetic_task(&task, input_id.clone(), task.needs.clone()); + input_boundary.input = values; + input_boundary.output_schema = Some(definition.input_schema.clone()); + tasks.push(input_boundary); + synthetic.insert( + input_id.clone(), + TaskUse::SubworkflowInput(CompiledSubworkflowInput { + name: name.to_owned(), + version: definition.version.clone(), + }), + ); + + stack.push(name.to_owned()); + for definition_task in &definition.tasks { + let mut child = definition_task.clone(); + child.id = local_ids[&definition_task.id].clone(); + child.needs = definition_task + .needs + .iter() + .map(|needed| { + local_ids + .get(needed) + .cloned() + .unwrap_or_else(|| needed.clone()) + }) + .collect(); + if !child.needs.contains(&input_id) { + child.needs.push(input_id.clone()); + } + rewrite_subworkflow_task( + &mut child, + &local_ids, + &input_id, + &format!("{}__", task.id.replace('-', "_")), + ); + namespace_subworkflow_action_state( + workflow, + &mut child, + &format!("{}__", task.id.replace('-', "_")), + file, + diagnostics, + ); + instantiate_subworkflow_task(workflow, child, file, stack, tasks, synthetic, diagnostics); + } + stack.pop(); + + let children = definition + .tasks + .iter() + .map(|child| local_ids[&child.id].clone()) + .collect::>(); + let outputs = rewrite_subworkflow_value( + &Value::Object(definition.outputs.clone().into_iter().collect()), + &local_ids, + &input_id, + &format!("{}__", task.id.replace('-', "_")), + ); + let mut aggregate_needs = children.clone(); + aggregate_needs.push(input_id); + let mut aggregate = synthetic_task(&task, task.id.clone(), aggregate_needs); + aggregate.input = outputs + .as_object() + .cloned() + .unwrap_or_default() + .into_iter() + .collect(); + aggregate.output_schema = Some(definition.output_schema.clone()); + tasks.push(aggregate); + synthetic.insert( + task.id.clone(), + TaskUse::SubworkflowAggregate(CompiledSubworkflowAggregate { + name: name.to_owned(), + version: definition.version.clone(), + children, + }), + ); +} + +fn synthetic_task(source: &TaskDefinition, id: String, needs: Vec) -> TaskDefinition { + let mut task = source.clone(); + task.id = id; + task.needs = needs; + task.foreach = None; + task.matrix = None; + task.route = None; + task.loop_definition = None; + task.memory_writes.clear(); + task.when = None; + task.vars.clear(); + task.input.clear(); + task.retry = RetryDefinition::default(); + task.timeout_seconds = None; + task.output_schema = None; + task +} + +fn namespace_subworkflow_action_state( + workflow: &Workflow, + task: &mut TaskDefinition, + prefix: &str, + file: &str, + diagnostics: &mut Vec, +) { + let Some(name) = task.uses.strip_prefix("action:") else { + return; + }; + let Some(action) = workflow.spec.actions.get(name) else { + return; + }; + let field = match action.kind { + ActionKind::MemoryRead | ActionKind::MemoryWrite => "key", + ActionKind::LongTermMemoryRead | ActionKind::LongTermMemoryWrite => "namespace", + _ => return, + }; + let Some(Value::String(value)) = task.input.get_mut(field) else { + return; + }; + if value.contains("${{") { + diagnostics.push(Diagnostic::error( + DiagnosticCode::UnsupportedCapability, + file, + format!( + "sub-workflow task `{}` requires a static `{field}` for isolated state", + task.id + ), + )); + } else { + value.insert_str(0, prefix); + } +} + +fn rewrite_subworkflow_task( + task: &mut TaskDefinition, + local_ids: &BTreeMap, + input_id: &str, + memory_prefix: &str, +) { + task.when = task + .when + .as_ref() + .map(|value| rewrite_subworkflow_string(value, local_ids, input_id, memory_prefix)); + task.vars = rewrite_subworkflow_map(&task.vars, local_ids, input_id, memory_prefix); + task.input = rewrite_subworkflow_map(&task.input, local_ids, input_id, memory_prefix); + task.memory_writes = task + .memory_writes + .iter() + .map(|key| format!("{memory_prefix}{key}")) + .collect(); + if let Some(foreach) = &mut task.foreach { + foreach.items = foreach + .items + .iter() + .map(|value| rewrite_subworkflow_value(value, local_ids, input_id, memory_prefix)) + .collect(); + } + if let Some(matrix) = &mut task.matrix { + for values in matrix.axes.values_mut() { + *values = values + .iter() + .map(|value| rewrite_subworkflow_value(value, local_ids, input_id, memory_prefix)) + .collect(); + } + } + if let Some(route) = &mut task.route { + route.select = + rewrite_subworkflow_string(&route.select, local_ids, input_id, memory_prefix); + for case in &mut route.cases { + case.tasks = case + .tasks + .iter() + .map(|id| local_ids.get(id).cloned().unwrap_or_else(|| id.clone())) + .collect(); + } + route.default = route + .default + .iter() + .map(|id| local_ids.get(id).cloned().unwrap_or_else(|| id.clone())) + .collect(); + } + if let Some(loop_definition) = &mut task.loop_definition { + loop_definition.condition = rewrite_subworkflow_string( + &loop_definition.condition, + local_ids, + input_id, + memory_prefix, + ); + loop_definition.initial = + rewrite_subworkflow_value(&loop_definition.initial, local_ids, input_id, memory_prefix); + } +} + +fn rewrite_subworkflow_map( + values: &JsonMap, + local_ids: &BTreeMap, + input_id: &str, + memory_prefix: &str, +) -> JsonMap { + values + .iter() + .map(|(key, value)| { + ( + key.clone(), + rewrite_subworkflow_value(value, local_ids, input_id, memory_prefix), + ) + }) + .collect() +} + +fn rewrite_subworkflow_value( + value: &Value, + local_ids: &BTreeMap, + input_id: &str, + memory_prefix: &str, +) -> Value { + match value { + Value::String(value) => Value::String(rewrite_subworkflow_string( + value, + local_ids, + input_id, + memory_prefix, + )), + Value::Array(values) => Value::Array( + values + .iter() + .map(|value| rewrite_subworkflow_value(value, local_ids, input_id, memory_prefix)) + .collect(), + ), + Value::Object(values) => Value::Object( + values + .iter() + .map(|(key, value)| { + ( + key.clone(), + rewrite_subworkflow_value(value, local_ids, input_id, memory_prefix), + ) + }) + .collect(), + ), + value => value.clone(), + } +} + +fn rewrite_subworkflow_string( + value: &str, + local_ids: &BTreeMap, + input_id: &str, + memory_prefix: &str, +) -> String { + let mut rewritten = value.replace("inputs.", &format!("tasks.{input_id}.output.")); + rewritten = rewritten.replace("memory.", &format!("memory.{memory_prefix}")); + for (local, namespaced) in local_ids { + rewritten = rewritten.replace(&format!("tasks.{local}."), &format!("tasks.{namespaced}.")); + } + rewritten +} + pub fn compile(workflow: &Workflow, file: &str) -> Result> { let mut diagnostics = Vec::new(); let mut tasks = BTreeMap::new(); let mut declaration_order = Vec::new(); let mut source_positions = BTreeMap::new(); - let expanded_tasks = expand_tasks(workflow, file, &mut diagnostics); + let (workflow, synthetic_uses) = expand_subworkflow_calls(workflow, file, &mut diagnostics); + let expanded_tasks = expand_tasks(&workflow, file, &mut diagnostics, &synthetic_uses); if !diagnostics.is_empty() { return Err(diagnostics); } @@ -691,17 +1095,23 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => JsonMap::new(), }; input.extend(task.input.clone()); let memory_writes = if matches!( task_use, - TaskUse::Aggregate(_) | TaskUse::Router(_) | TaskUse::LoopAggregate(_) + TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) ) { Vec::new() } else { task_memory_writes( - workflow, + &workflow, &task_use, &input, &task.memory_writes, @@ -720,7 +1130,9 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => JsonMap::new(), }; vars.extend(task.vars.clone()); declaration_order.push(task.id.clone()); @@ -781,8 +1193,8 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result ")), )] })?; - validate_parallel_memory_writes(workflow, &order, &tasks, file, &mut diagnostics); + validate_parallel_memory_writes(&workflow, &order, &tasks, file, &mut diagnostics); if !diagnostics.is_empty() { return Err(diagnostics); } @@ -812,6 +1224,9 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result PlanPredictability::FullyPredictable, TaskUse::Router(_) => PlanPredictability::FullyPredictable, TaskUse::LoopAggregate(_) => PlanPredictability::FullyPredictable, + TaskUse::SubworkflowInput(_) | TaskUse::SubworkflowAggregate(_) => { + PlanPredictability::FullyPredictable + } }; } let predictability = tasks @@ -823,7 +1238,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result 2, }) .unwrap_or(PlanPredictability::FullyPredictable); - let workflow_json = serde_json::to_vec(workflow).map_err(|error| { + let workflow_json = serde_json::to_vec(&workflow).map_err(|error| { vec![Diagnostic::error( DiagnosticCode::SchemaViolation, file, @@ -831,7 +1246,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result false, + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => false, }; if !is_memory_write { if !declared.is_empty() { @@ -1272,7 +1689,11 @@ fn plan_requirements( predictability: task.predictability, }] } - TaskUse::Aggregate(_) | TaskUse::Router(_) | TaskUse::LoopAggregate(_) => Vec::new(), + TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => Vec::new(), }) .collect(); PlanRequirements { @@ -2183,6 +2604,198 @@ spec: })); } + #[test] + fn subworkflow_compiles_to_typed_namespaced_boundaries() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: subworkflow } +spec: + inputs: { message: hello } + actions: + assign: { kind: builtin.assign } + subworkflows: + summarize: + version: 1.2.0 + inputs: { message: default } + inputSchema: + type: object + required: [message] + additionalProperties: false + properties: { message: { type: string } } + outputs: + result: "${{ tasks.second.output.output.value }}" + outputSchema: + type: object + required: [result] + additionalProperties: false + properties: { result: { type: string } } + tasks: + - { id: first, uses: "action:assign", with: { value: "${{ inputs.message }}" } } + - id: second + uses: action:assign + needs: [first] + with: { value: "${{ tasks.first.output.output.value }}" } + tasks: + - id: summary + uses: workflow:summarize + with: { message: "${{ inputs.message }}" } + - { id: done, uses: "action:assign", needs: [summary] } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("sub-workflow compiles"); + let second = compile(&workflow, "fixture.yaml").expect("stable expansion"); + assert_eq!(plan, second); + assert_eq!(plan.order.len(), 5); + let input_id = plan + .order + .iter() + .find(|id| id.starts_with("summary--inputs-")) + .expect("input boundary") + .clone(); + assert!(matches!( + plan.tasks[&input_id].uses, + TaskUse::SubworkflowInput(_) + )); + assert_eq!( + plan.tasks["summary--first"].input["value"], + Value::String(format!("${{{{ tasks.{input_id}.output.message }}}}")) + ); + assert_eq!( + plan.tasks["summary--second"].needs, + ["summary--first".to_owned(), input_id.clone()] + ); + let aggregate = match &plan.tasks["summary"].uses { + TaskUse::SubworkflowAggregate(aggregate) => aggregate, + other => panic!("expected sub-workflow aggregate, got {other:?}"), + }; + assert_eq!(aggregate.name, "summarize"); + assert_eq!(aggregate.version, "1.2.0"); + assert_eq!(aggregate.children, ["summary--first", "summary--second"]); + assert_eq!( + plan.tasks["summary"].needs, + [ + "summary--first".to_owned(), + "summary--second".to_owned(), + input_id.clone() + ] + ); + assert_eq!( + plan.tasks["summary"].input["result"], + "${{ tasks.summary--second.output.output.value }}" + ); + } + + #[test] + fn subworkflow_rejects_cycles_and_invalid_versions() { + let cycle = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: subworkflow-cycle } +spec: + subworkflows: + first: + version: 1.0.0 + inputSchema: { type: object } + outputSchema: { type: object } + tasks: [{ id: nested, uses: "workflow:second" }] + second: + version: 1.0.0 + inputSchema: { type: object } + outputSchema: { type: object } + tasks: [{ id: nested, uses: "workflow:first" }] + tasks: [{ id: invoke, uses: "workflow:first" }] +"#, + ); + let diagnostics = compile(&cycle, "fixture.yaml").expect_err("cycle rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("sub-workflow cycle: first -> second -> first") + })); + + let invalid = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: invalid-version } +spec: + subworkflows: + broken: + version: latest + inputSchema: { type: object } + outputSchema: { type: object } + tasks: [] + tasks: [{ id: invoke, uses: "workflow:broken" }] +"#, + ); + let diagnostics = compile(&invalid, "fixture.yaml").expect_err("version rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("version must be semantic versioning") + })); + + let output_override = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: output-override } +spec: + subworkflows: + typed: + version: 1.0.0 + inputSchema: { type: object } + outputSchema: { type: object } + tasks: [] + tasks: + - id: invoke + uses: workflow:typed + outputSchema: { type: string } +"#, + ); + let diagnostics = compile(&output_override, "fixture.yaml").expect_err("override rejected"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("accepts needs, with, and failure only") + })); + } + + #[test] + fn subworkflow_working_memory_is_namespaced_per_invocation() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: isolated-subworkflows } +spec: + runtime: { maxConcurrency: 2 } + actions: + remember: { kind: builtin.memory.write } + subworkflows: + remember: + version: 1.0.0 + inputSchema: { type: object } + outputSchema: { type: object } + tasks: + - id: write + uses: action:remember + with: { key: result, value: stored } + tasks: + - { id: left, uses: "workflow:remember" } + - { id: right, uses: "workflow:remember" } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("isolated calls compile"); + assert_eq!(plan.tasks["left--write"].memory_writes, ["left__result"]); + assert_eq!(plan.tasks["right--write"].memory_writes, ["right__result"]); + assert_eq!(plan.tasks["left--write"].input["key"], "left__result"); + assert_eq!(plan.tasks["right--write"].input["key"], "right__result"); + } + #[test] fn typed_router_cases_compile_to_explicit_destination_guards() { let workflow = parse( diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index 6694764..1ebf0f8 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -50,6 +50,8 @@ pub struct WorkflowSpec { pub actions: BTreeMap, #[serde(default)] pub tools: BTreeMap, + #[serde(default)] + pub subworkflows: BTreeMap, pub tasks: Vec, #[serde(default)] pub policy: PolicyDefinition, @@ -446,6 +448,19 @@ pub struct TaskDefinition { pub output_schema: Option, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubworkflowDefinition { + pub version: String, + #[serde(default)] + pub inputs: JsonMap, + pub input_schema: Value, + #[serde(default)] + pub outputs: JsonMap, + pub output_schema: Value, + pub tasks: Vec, +} + pub const DEFAULT_MAX_EXPANSION_ITEMS: usize = 32; pub const MAX_EXPANSION_ITEMS: usize = 256; diff --git a/crates/agentctl-core/src/pack.rs b/crates/agentctl-core/src/pack.rs index 377915a..488181c 100644 --- a/crates/agentctl-core/src/pack.rs +++ b/crates/agentctl-core/src/pack.rs @@ -8,7 +8,9 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use thiserror::Error; -use crate::dsl::{ActionDefinition, AgentDefinition, PolicyDefinition, ToolDefinition}; +use crate::dsl::{ + ActionDefinition, AgentDefinition, PolicyDefinition, SubworkflowDefinition, ToolDefinition, +}; pub const PACK_API_VERSION: &str = "agentctl.dev/pack/v1alpha1"; @@ -26,6 +28,8 @@ pub struct PackManifest { #[serde(default)] pub tools: BTreeMap, #[serde(default)] + pub workflows: BTreeMap, + #[serde(default)] pub capabilities: Vec, #[serde(default)] pub providers: Vec, @@ -66,6 +70,21 @@ impl PackManifest { )) })?; } + for (name, workflow) in &self.workflows { + Version::parse(&workflow.version).map_err(|error| { + PackError::Invalid(format!("workflow `{name}` version is not semver: {error}")) + })?; + for (label, schema) in [ + ("inputSchema", &workflow.input_schema), + ("outputSchema", &workflow.output_schema), + ] { + jsonschema::validator_for(schema).map_err(|error| { + PackError::Invalid(format!( + "workflow `{name}` {label} is not valid JSON Schema: {error}" + )) + })?; + } + } Ok(()) } } diff --git a/crates/agentctl-core/tests/compatibility.rs b/crates/agentctl-core/tests/compatibility.rs index 6278106..8cfbcef 100644 --- a/crates/agentctl-core/tests/compatibility.rs +++ b/crates/agentctl-core/tests/compatibility.rs @@ -42,7 +42,9 @@ fn typescript_assign_fixture_translates_to_the_language_neutral_contract() { TaskUse::Agent(_) | TaskUse::Aggregate(_) | TaskUse::Router(_) - | TaskUse::LoopAggregate(_) => { + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => { panic!("expected action task") } } diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index a856850..fb005a5 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -2462,14 +2462,17 @@ impl Runtime { { continue; } - if !matches!(task.uses, TaskUse::Aggregate(_) | TaskUse::LoopAggregate(_)) - && dependencies.iter().any(|dependency| { - matches!( - dependency.state, - TaskState::Failed | TaskState::Cancelled | TaskState::Skipped - ) - }) - { + if !matches!( + task.uses, + TaskUse::Aggregate(_) + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowAggregate(_) + ) && dependencies.iter().any(|dependency| { + matches!( + dependency.state, + TaskState::Failed | TaskState::Cancelled | TaskState::Skipped + ) + }) { self.store.transition_task( run_id, task_id, @@ -3002,14 +3005,17 @@ impl Runtime { .iter() .filter_map(|needed| tasks.iter().find(|candidate| &candidate.task_id == needed)) .collect(); - if !matches!(task.uses, TaskUse::Aggregate(_) | TaskUse::LoopAggregate(_)) - && dependencies.iter().any(|dependency| { - matches!( - dependency.state, - TaskState::Failed | TaskState::Cancelled | TaskState::Skipped - ) - }) - { + if !matches!( + task.uses, + TaskUse::Aggregate(_) + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowAggregate(_) + ) && dependencies.iter().any(|dependency| { + matches!( + dependency.state, + TaskState::Failed | TaskState::Cancelled | TaskState::Skipped + ) + }) { self.store.transition_task( run_id, &task.id, @@ -3516,6 +3522,12 @@ impl Runtime { memory: None, }) } + TaskUse::SubworkflowInput(_) | TaskUse::SubworkflowAggregate(_) => { + Ok(TaskExecution::Complete { + output: input, + memory: None, + }) + } } } @@ -5009,6 +5021,7 @@ fn task_output_schema(workflow: &Workflow, task: &agentctl_core::CompiledTask) - }, "additionalProperties": false })), + TaskUse::SubworkflowInput(_) | TaskUse::SubworkflowAggregate(_) => None, }) } @@ -5108,6 +5121,16 @@ fn task_definition_fingerprint( "task": task, "loop": loop_aggregate, }), + TaskUse::SubworkflowInput(input) => serde_json::json!({ + "kind": "subworkflow_input", + "task": task, + "subworkflow": input, + }), + TaskUse::SubworkflowAggregate(aggregate) => serde_json::json!({ + "kind": "subworkflow_aggregate", + "task": task, + "subworkflow": aggregate, + }), }; versioned_json_digest(&serde_json::json!({ "formatVersion": 1, @@ -10210,6 +10233,258 @@ spec: ); } + #[tokio::test] + async fn subworkflow_boundaries_support_typed_execution_retry_repair_and_replay() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: subworkflow-runtime } +spec: + inputs: { message: hello } + outputs: { result: "${{ tasks.summary.output.result }}" } + actions: + assign: { kind: builtin.assign } + subworkflows: + summarize: + version: 1.0.0 + inputSchema: + type: object + required: [message] + additionalProperties: false + properties: { message: { type: string } } + outputSchema: + type: object + required: [result] + additionalProperties: false + properties: { result: { type: string } } + outputs: + result: "${{ tasks.second.output.output.value }}" + tasks: + - { id: first, uses: "action:assign", with: { value: "${{ inputs.message }}" } } + - id: second + uses: action:assign + needs: [first] + with: { value: "${{ tasks.first.output.output.value }}" } + tasks: + - id: summary + uses: workflow:summarize + with: { message: "${{ inputs.message }}" } +"#, + ); + let source = runtime + .start( + &workflow, + &plan, + serde_json::json!({"message": "hello"}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("sub-workflow succeeds"); + assert_eq!(source.state, RunState::Succeeded); + assert_eq!(source.output.as_ref().expect("output")["result"], "hello"); + let source_tasks = store.list_tasks(&source.run_id).expect("tasks"); + assert_eq!(source_tasks.len(), 4); + assert!( + source_tasks + .iter() + .any(|task| task.task_id.starts_with("summary--inputs-")) + ); + + let invalid_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({"message": 7}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { + run_id, + task, + message, + .. + }) => { + assert!(task.starts_with("summary--inputs-")); + assert!(message.contains("task output contract failed")); + run_id + } + other => panic!("expected typed input failure, got {other:?}"), + }; + assert!( + store + .list_effects(&invalid_run_id) + .expect("invalid effects") + .is_empty() + ); + + let retry_plan = runtime + .plan_retry( + &source.run_id, + &workflow, + &plan, + &["summary--second".to_owned()], + false, + true, + ) + .expect("retry plan"); + assert!(retry_plan.compatible, "{:?}", retry_plan.blocked_reuse); + assert!( + retry_plan + .reused_tasks + .contains(&"summary--first".to_owned()) + ); + assert!(retry_plan.rerun_tasks.contains(&"summary".to_owned())); + let retried = runtime + .retry( + &workflow, + &plan, + retry_plan, + Some("retry namespaced boundary"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("retry"); + assert_eq!(retried.state, RunState::Succeeded); + + let repair_plan = runtime + .plan_repair( + &source.run_id, + &workflow, + &plan, + &["summary--second".to_owned()], + true, + ) + .expect("repair plan"); + assert!(repair_plan.compatible, "{:?}", repair_plan.blocked_reuse); + let repaired = runtime + .repair( + &workflow, + &plan, + repair_plan, + Some("repair namespaced boundary"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("repair"); + assert_eq!(repaired.state, RunState::Succeeded); + let replay = runtime + .replay(&repaired.run_id) + .await + .expect("sub-workflow replay"); + assert_eq!(replay.state, RunState::Succeeded); + assert_eq!(replay.output, repaired.output); + assert!( + store + .list_effects(&replay.run_id) + .expect("replay effects") + .is_empty() + ); + } + + #[tokio::test] + async fn subworkflow_children_own_artifacts_and_report_namespaced_failures() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: subworkflow-artifact } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + subworkflows: + publish: + version: 1.0.0 + inputSchema: { type: object, additionalProperties: false } + outputSchema: { type: object, additionalProperties: false } + tasks: + - id: write + uses: action:write + with: { path: subworkflow.txt, content: durable } + tasks: + - { id: publish, uses: "workflow:publish" } +"#, + ); + let outcome = runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("artifact sub-workflow succeeds"); + let tasks = store.list_tasks(&outcome.run_id).expect("tasks"); + assert_eq!( + tasks + .iter() + .find(|task| task.task_id == "publish--write") + .expect("artifact child") + .artifact_manifest + .len(), + 1 + ); + assert!( + tasks + .iter() + .find(|task| task.task_id == "publish") + .expect("aggregate") + .artifact_manifest + .is_empty() + ); + + let (failed_workflow, failed_plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: subworkflow-failure } +spec: + actions: + assert: { kind: builtin.assert } + subworkflows: + verify: + version: 1.0.0 + inputSchema: { type: object } + outputSchema: { type: object } + tasks: + - { id: check, uses: "action:assert", with: { that: false } } + tasks: + - { id: verify, uses: "workflow:verify" } +"#, + ); + match runtime + .start( + &failed_workflow, + &failed_plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { task, .. }) => { + assert_eq!(task, "verify--check"); + } + other => panic!("expected namespaced child failure, got {other:?}"), + } + } + #[tokio::test] async fn task_output_contract_failure_is_durable() { let directory = tempdir().expect("tempdir"); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 69be5ea..f8dca98 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -39,9 +39,11 @@ routers are pure tasks whose enumerated destination guards compile into the graph; condition and route decisions are durable and replayable. Bounded loops compile into sequential namespaced iteration tasks and a pure aggregate, so iteration attempts, effects, guard decisions, retry, repair, and replay use the -ordinary durable task model. Sub-workflows, handlers, compensation execution, -and event triggers still require their own explicit state and recovery -contracts. The DSL carries +ordinary durable task model. Reusable sub-workflows compile into a typed input +boundary, namespaced ordinary tasks, and a typed output aggregate. Their policy +and providers come from the invoking workflow, while deterministic memory keys +are invocation-prefixed. Handlers, compensation execution, and event triggers +still require their own explicit state and recovery contracts. The DSL carries optional compensation metadata on a tool contract, but the runtime does not execute compensation. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 4b76a4d..0e78e6a 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -22,4 +22,4 @@ Legacy workflows depending on packs, broad built-in tool profiles, remote MCP/A2 ## Separate product decisions -Sub-workflows, teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. Bounded loops are additive; unbounded or model-controlled iteration is intentionally unsupported. +Teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. Bounded loops and namespaced sub-workflows are additive; unbounded or model-controlled graph growth is intentionally unsupported. diff --git a/docs/DSL.md b/docs/DSL.md index 1a3b026..c419bc5 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,7 +2,7 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach`, `matrix`, or `loop` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; reusable sub-workflows; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, `workflow:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach`, `matrix`, or `loop` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. @@ -37,8 +37,14 @@ initial value or preceding iteration output. A still-true guard after the maximum fails closed. Retry and repair select iteration IDs. See [Bounded loops](guides/BOUNDED_LOOPS.md). +Reusable `subworkflows` declare a semantic version, input/output JSON Schemas, +default input values, an output map, and local tasks. An invocation compiles to +a typed input boundary, `INVOCATION--LOCAL_TASK` children, and a typed output +aggregate. Pack manifests export the same contract under `workflows`. See +[Reusable sub-workflows](guides/SUB_WORKFLOWS.md). + `builtin.shell.exec` captures stdout and stderr concurrently. Its optional `stdoutLimitBytes`, `stderrLimitBytes`, and `combinedOutputLimitBytes` fields default to 1 MiB, 1 MiB, and 2 MiB respectively. Each configured value must be between 1 byte and 16 MiB. `timeoutSeconds` must be between 1 and 86,400. Exceeding an output bound terminates and reaps the process and records a structured failed effect; timeout or cancellation remains an uncertain effect because external changes may already have occurred. These fields are validated identically for workflow and pack actions. The parser translates a limited unversioned `playbook:` document and emits a migration warning. Use `agentctl migrate old.yaml --write new.yaml`. Legacy pack-backed, MCP, A2A, provider-specific, and broad module configurations need manual migration; see [Migrating from TypeScript](MIGRATING_FROM_TYPESCRIPT.md). -Not implemented in v1alpha1: sub-workflows, `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. +Not implemented in v1alpha1: `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index f3f7e65..762190f 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -48,4 +48,9 @@ pure aggregate. Each iteration retains its guard decision, output, effects, artifacts, attempts, and recovery identity. A false guard skips the remaining chain. A guard that remains true after the declared maximum fails closed. +Sub-workflows also compile before run creation. Typed input and output boundary +tasks surround namespaced child tasks, so child attempts, effects, artifacts, +approvals, retry/repair lineage, cancellation, and replay stay in the ordinary +run graph. + The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 103d23f..e1124ac 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -23,7 +23,7 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an These are useful extensions but are not required by the product thesis. They need new deterministic state and compatibility contracts before implementation: -- sub-workflows; compensation execution; +- compensation execution; - structured agent teams and handoffs; - model token streaming into CLI/workflow state; - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; @@ -53,6 +53,9 @@ These are useful extensions but are not required by the product thesis. They nee - Loops are sequential, require a maximum from 1 through 64, and compile all iteration boundaries before execution. Runtime or model-controlled graph growth and unbounded loops are rejected. +- Sub-workflows are compile-time namespaced graphs with semantic versions and + typed input/output boundaries. Definitions inherit the caller's policy and + providers and cannot request independent authority. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. diff --git a/docs/PACKS.md b/docs/PACKS.md index 661e89d..8213a09 100644 --- a/docs/PACKS.md +++ b/docs/PACKS.md @@ -4,4 +4,8 @@ A pack is reviewed reusable YAML content, not executable plugin code. Its manife `agentctl packs inspect` strictly parses the manifest, validates the API version, name, versions, and compatibility with the running binary. `agentctl packs verify` compares a `sha256:` integrity digest for a local manifest or archive. Workflow pack references carry name, version, local path, and integrity. The CLI verifies a referenced manifest, keeps it beneath the workflow directory, and loads actions, agents, and tools as `.` before compilation. -Dependency resolution, transitive lockfile generation, Git fetching, reusable sub-workflows, policy-default merging, a hosted registry, native dynamic libraries, and pack processes are not implemented. A checked-in pack reference is therefore an integrity/provenance contract for local content, not a package manager. Manifest policy defaults are inspectable metadata and never weaken the invoking workflow’s policy. +Dependency resolution, transitive lockfile generation, Git fetching, policy-default merging, a hosted registry, native dynamic libraries, and pack processes are not implemented. A checked-in pack reference is therefore an integrity/provenance contract for local content, not a package manager. Manifest policy defaults are inspectable metadata and never weaken the invoking workflow’s policy. + +A pack may export typed reusable definitions under `workflows`. Their actions, +agents, tools, and nested workflow references are pack-qualified during secure +loading. The invoking workflow still supplies policy and configured providers. diff --git a/docs/adr/0012-subworkflows-as-namespaced-graphs.md b/docs/adr/0012-subworkflows-as-namespaced-graphs.md new file mode 100644 index 0000000..73e0e25 --- /dev/null +++ b/docs/adr/0012-subworkflows-as-namespaced-graphs.md @@ -0,0 +1,26 @@ +# ADR 0012: sub-workflows as namespaced graphs + +Status: accepted + +## Decision + +Reusable sub-workflows compile into the invoking plan. Each invocation creates +one typed input boundary, namespaced ordinary child tasks, and one typed output +aggregate. Definitions carry a semantic version and JSON Schemas for their +input and output interfaces. Pack definitions are covered by the existing +version and integrity pin. + +The invoking workflow supplies policy and providers. Definitions cannot widen +policy. Deterministic memory state is invocation-prefixed, while effects, +artifacts, attempts, approvals, audit records, and traces remain owned by the +expanded child task. + +## Consequences + +- Scheduling, cancellation, failure propagation, retry, repair, replay, and + inspection use the existing durable task model. +- Nested definitions are flattened recursively and cycles fail compilation. +- No second runtime, database, effect ledger, or hidden provider session exists. +- Namespaced IDs are part of recovery and inspection contracts. +- Dynamic memory keys are rejected because compile-time isolation cannot be + proven. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 77fc4c1..1a23ef8 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -78,7 +78,7 @@ cargo xtask acceptance-container | Foreach/matrix | compiler bounds/identity tests and runtime partial-failure, child retry, sibling reuse, aggregation, and replay tests passed | packaged CLI scenario 34 passed | deterministic verified; live pending | | Conditions/routers | compiler typed-case/guard failures and runtime durable condition, route, retry, changed-input repair, and skipped replay tests passed | packaged CLI scenario 35 passed | deterministic verified; live pending | | Bounded loops | compiler bounds/identity tests and runtime zero/one/max, exhaustion, cancellation, uncertain effect, retry, repair, and replay tests passed | packaged CLI scenario 36 passed | deterministic verified; live pending | -| Sub-workflows | pending | pending | open | +| Sub-workflows | compiler namespacing/version/cycle/state-isolation tests and runtime typed boundary, retry, repair, and replay tests passed | packaged CLI scenario 37 and integrity-pinned pack example passed | deterministic verified; live pending | | Compensation/handoffs/streaming | pending | pending | open | | MCP/A2A resilience | pending | pending | open | | Packs/trust/extensions | pending | pending | open | diff --git a/docs/execution/DECISIONS.md b/docs/execution/DECISIONS.md index 5649499..450f0b8 100644 --- a/docs/execution/DECISIONS.md +++ b/docs/execution/DECISIONS.md @@ -13,5 +13,6 @@ | [0009](../adr/0009-bounded-static-task-expansion.md) | Bounded static task expansion | accepted | Stable child tasks and ordered aggregates prevent model-controlled graph growth. | | [0010](../adr/0010-typed-routing-and-durable-decisions.md) | Typed routing and durable decisions | accepted | Pure enumerated routers and hashed condition contexts make branching inspectable and replayable. | | [0011](../adr/0011-bounded-loops-as-static-graphs.md) | Bounded loops as static graphs | accepted | Fixed iteration chains reuse ordinary durable task and recovery semantics. | +| [0012](../adr/0012-subworkflows-as-namespaced-graphs.md) | Sub-workflows as namespaced graphs | accepted | Typed boundaries and flattened children avoid a hidden nested runtime. | These decisions resolve the researched patterns in [LANDSCAPE.md](../research/LANDSCAPE.md). No unsafe code or distributed control plane ADR is required because neither exists. diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md index e35f27a..16c07d0 100644 --- a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -47,6 +47,7 @@ This inventory is enforced by `cargo xtask examples-verify`. The default command | `examples/v1/reusable-pack.yaml` | Native reusable pack consumer | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | Pack digest | Canonical | passed | | `examples/v1/router.yaml` | Typed deterministic route selection | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Decision and skipped branch | passed | | `examples/v1/secret-reference.yaml` | Environment reference contract | OpenAI | success | 0 | 0 | N/A | Protocol mock | Passed 2026-07-23 | N/A | N/A | Secret-safe live gate | live passed | +| `examples/v1/subworkflow.yaml` | Typed reusable namespaced graph | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Typed input/output boundaries | passed | | `examples/v1/working-memory.yaml` | Working-memory update | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | SQLite | Canonical | passed | | `fixtures/compat/v0/assign.playbook.yaml` | Language-neutral TypeScript compatibility fixture | legacy translator | success | 0 | 0 | Compatibility test | N/A | N/A | N/A | N/A | Oracle contract | passed | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index 8bdb023..e84d825 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -47,7 +47,7 @@ complete, every entry must have exactly one final disposition: | DYN-001 | Foreach and matrix | in progress | implemented | | COND-001 | Conditions and routers | in progress | implemented | | LOOP-001 | Bounded loops | in progress | implemented | -| SUB-001 | Sub-workflows | open | implemented | +| SUB-001 | Sub-workflows | in progress | implemented | | COMP-001 | Compensation | open | implemented | | TEAM-001 | Structured teams and handoffs | open | redesigned | | STR-001 | Streaming | open | implemented | @@ -318,7 +318,9 @@ complete, every entry must have exactly one final disposition: ### SUB-001: Reusable sub-workflows -- Current behavior: packs can contribute actions/agents/tools but not workflows. +- Current behavior: inline and integrity-pinned pack definitions compile into a + typed input boundary, namespaced ordinary child tasks, and a typed output + aggregate. - User impact: reusable graph composition requires copying tasks. - Security or durability impact: implicit policy/provider inheritance could broaden authority. @@ -327,13 +329,24 @@ complete, every entry must have exactly one final disposition: - Required implementation: pack/local definitions, namespace escaping, recursion/cycle checks, state isolation, provider mapping, artifact ownership, lineage, errors, inspection, repair/retry/replay. -- Migration impact: pack/lock, workflow schema, and plan format. +- Migration impact: workflow and pack schemas gain additive reusable workflow + definitions. Compiled plans gain pure boundary variants; existing task, + effect, artifact, checkpoint, and attempt storage is reused. - Tests: nesting, collisions, cycles, policy narrowing, output contracts, failures, artifacts, repair/retry/replay. - Examples: operational workflow calling a reusable sub-workflow. - Live evidence: sub-workflow containing one OpenAI task. - Documentation: authoring, versioning, and policy inheritance. -- Final disposition: pending implementation evidence. +- Final disposition: implemented. Definitions carry a semantic version and + JSON Schema input/output interfaces. Nested calls flatten recursively and + cycles fail compilation. The caller's policy and providers remain + authoritative, deterministic memory keys are invocation-prefixed, and + namespaced children retain artifact/effect ownership and ordinary + retry/repair/replay lineage. Focused compiler and runtime verification covers + stable expansion, typed rejection, state isolation, selected-boundary retry + and repair, and zero-effect replay. Packaged CLI scenario 37 and the + integrity-pinned pack example pass. Program state remains in progress until + the bounded live sub-workflow scenario executes. ### COMP-001: Explicit compensation diff --git a/docs/guides/SUB_WORKFLOWS.md b/docs/guides/SUB_WORKFLOWS.md new file mode 100644 index 0000000..50f4df1 --- /dev/null +++ b/docs/guides/SUB_WORKFLOWS.md @@ -0,0 +1,72 @@ +# Reusable sub-workflows + +Sub-workflows package a typed graph behind one invocation task. Compilation +expands the graph into ordinary namespaced tasks, a durable typed input +boundary, and a typed output aggregate. There is no hidden nested scheduler. + +```yaml +subworkflows: + summarize: + version: 1.0.0 + inputSchema: + type: object + required: [message] + additionalProperties: false + properties: + message: { type: string } + outputSchema: + type: object + required: [result] + additionalProperties: false + properties: + result: { type: string } + outputs: + result: "${{ tasks.finish.output.output.result }}" + tasks: + - id: finish + uses: action:assign + with: + result: "${{ inputs.message }}" + +tasks: + - id: summary + uses: workflow:summarize + with: + message: durable +``` + +`version` must be semantic versioning. The invocation input is rendered once, +validated against `inputSchema`, and stored as a normal task output. References +to the definition's `inputs` read that boundary. The output map is rendered +after the child graph and validated against `outputSchema`. + +Compiled IDs use `INVOCATION--LOCAL_TASK`. Input boundaries use a stable +digest-qualified `INVOCATION--inputs-DIGEST` ID. Nested sub-workflows expand +recursively. Cycles, missing local dependencies, invalid schemas, and ID +collisions fail compilation. + +## Inheritance and isolation + +The invoking workflow's policy is authoritative. A sub-workflow cannot add +policy grants. Providers are resolved from the invoking workflow, while +pack-provided agents, actions, tools, and sub-workflows receive integrity-pinned +pack-qualified names. + +Working-memory keys and long-term-memory namespaces used by deterministic +memory actions are prefixed per invocation. Dynamic memory keys are rejected +because they cannot prove isolation before execution. Child effects, artifacts, +approvals, traces, and audit records retain their namespaced child task owner. + +## Recovery + +Every expanded child is an ordinary durable task. Retry and repair select the +visible namespaced IDs and reuse compatible predecessors. Failures report the +namespaced boundary. Recorded replay copies input, child, and output boundary +records without dispatching providers, tools, processes, or network calls. + +Pack manifests may export definitions under `workflows`. A workflow pins the +pack name, version, path, and integrity digest, then invokes +`workflow:PACK_NAME.DEFINITION`. + +See [`examples/v1/subworkflow.yaml`](../../examples/v1/subworkflow.yaml) and +[`examples/v1/reusable-pack.yaml`](../../examples/v1/reusable-pack.yaml). diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index 9764b84..42b1bdb 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -25,6 +25,7 @@ Unknown fields fail. Documents, ordinary input files, packs, direct reads, exist | `agents` | `{}` | Named bounded model executors. | | `actions` | `{}` | Named deterministic or protocol actions. | | `tools` | `{}` | Strict model-callable tool contracts. | +| `subworkflows` | `{}` | Semantically versioned reusable task graphs with typed input and output boundaries. | | `tasks` | required list | Ordered graph nodes. | | `policy` | safe defaults | Filesystem, process, network, provider, tool, and approval rules. | | `memory` | empty | Initial working memory and optional SQLite long-term namespace. | @@ -36,8 +37,8 @@ Unknown fields fail. Documents, ordinary input files, packs, direct reads, exist ## Tasks -Each task requires `id` and `uses`. `uses` is `action:name`, `agent:name`, or -`router`. +Each task requires `id` and `uses`. `uses` is `action:name`, `agent:name`, +`workflow:name`, or `router`. | Field | Default | Validation | | --- | --- | --- | @@ -59,8 +60,9 @@ Ready tasks are selected in YAML declaration order up to `maxConcurrency`. They read isolated durable snapshots and commit in compiled order. There is no runtime or model-controlled expansion. Static `foreach` and `matrix` tasks compile to inspectable child tasks and a parent aggregate. Bounded loops -compile to a sequential child chain and parent aggregate. There is no -sub-workflow, handler, or separate parallel group in this version. +compile to a sequential child chain and parent aggregate. Sub-workflows compile +to namespaced ordinary tasks with typed input and output boundaries. There is +no handler or separate parallel group in this version. ## Agents @@ -127,6 +129,7 @@ agentctl run examples/v1/dataflow.yaml --db /tmp/dataflow.db --output json --col Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Matrix and foreach](../guides/MATRIX_AND_FOREACH.md), [Conditions and routers](../guides/CONDITIONS_AND_ROUTERS.md), [Bounded -loops](../guides/BOUNDED_LOOPS.md), [Secret +loops](../guides/BOUNDED_LOOPS.md), [Reusable +sub-workflows](../guides/SUB_WORKFLOWS.md), [Secret references](../guides/SECRET_REFERENCES.md), [Policies](../policies.md), [Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). diff --git a/examples/v1/README.md b/examples/v1/README.md index 7e186e2..f88af18 100644 --- a/examples/v1/README.md +++ b/examples/v1/README.md @@ -13,10 +13,11 @@ The deterministic examples are exercised by `cargo xtask verify` and never requi - `parallel.yaml`: bounded parallel batches with disjoint working-memory writes and stable commits. - `matrix.yaml`: bounded static matrix expansion, stable child identities, and ordered aggregation. - `loop.yaml`: bounded sequential iteration, stable boundaries, aggregation, and fail-closed exhaustion. +- `subworkflow.yaml`: typed reusable graph expansion and namespaced recovery boundaries. - `working-memory.yaml` and `long-term-memory.yaml`: separate memory lifecycles. - `fake-provider.yaml`: deterministic model-provider path. - `mcp.yaml` and `a2a.yaml`: local protocol fixtures, backed by the protocol crate's mock-server tests. -- `example.pack.yaml` and `reusable-pack.yaml`: integrity-pinned local pack and executed packed action. +- `example.pack.yaml` and `reusable-pack.yaml`: integrity-pinned local pack with an executed packed action and sub-workflow. - `secret-reference.yaml`: environment reference without inline secret material. - `capability-failure.yaml`: a workflow expected to fail during compilation. diff --git a/examples/v1/example.pack.yaml b/examples/v1/example.pack.yaml index 71043fd..c58c232 100644 --- a/examples/v1/example.pack.yaml +++ b/examples/v1/example.pack.yaml @@ -7,3 +7,27 @@ providers: [] actions: assign: kind: builtin.assign +workflows: + echo: + version: 1.0.0 + inputSchema: + type: object + required: [value] + additionalProperties: false + properties: + value: + type: string + outputSchema: + type: object + required: [value] + additionalProperties: false + properties: + value: + type: string + outputs: + value: "${{ tasks.assign.output.output.value }}" + tasks: + - id: assign + uses: action:assign + with: + value: "${{ inputs.value }}" diff --git a/examples/v1/reusable-pack.yaml b/examples/v1/reusable-pack.yaml index cad43c4..bc3620c 100644 --- a/examples/v1/reusable-pack.yaml +++ b/examples/v1/reusable-pack.yaml @@ -3,13 +3,20 @@ kind: Workflow metadata: name: reusable-pack spec: + outputs: + workflowValue: "${{ tasks.packed-workflow.output.value }}" packs: - name: example.utility version: 1.0.0 path: example.pack.yaml - integrity: sha256:a010bcf3aca472351a2a63f097a6972423d98da007ef93a52ea47d284808a91a + integrity: sha256:a9b4d56e33524e73b109a66373e60e207516a7834c28e841e0accd456162a9ab tasks: - id: packed uses: action:example.utility.assign with: source: verified-pack + - id: packed-workflow + uses: workflow:example.utility.echo + needs: [packed] + with: + value: verified-sub-workflow diff --git a/examples/v1/subworkflow.yaml b/examples/v1/subworkflow.yaml new file mode 100644 index 0000000..33139c7 --- /dev/null +++ b/examples/v1/subworkflow.yaml @@ -0,0 +1,48 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: reusable-sub-workflow +spec: + inputs: + message: durable + outputs: + result: "${{ tasks.summary.output.result }}" + policy: + approval: never + actions: + assign: + kind: builtin.assign + subworkflows: + summarize: + version: 1.0.0 + inputSchema: + type: object + required: [message] + additionalProperties: false + properties: + message: + type: string + outputSchema: + type: object + required: [result] + additionalProperties: false + properties: + result: + type: string + outputs: + result: "${{ tasks.finish.output.output.result }}" + tasks: + - id: prepare + uses: action:assign + with: + value: "${{ inputs.message }}" + - id: finish + uses: action:assign + needs: [prepare] + with: + result: "${{ tasks.prepare.output.output.value }}" + tasks: + - id: summary + uses: workflow:summarize + with: + message: "${{ inputs.message }}" diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index 5183f9c..b5e55da 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -95,6 +95,13 @@ }, "default": {} }, + "subworkflows": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/SubworkflowDefinition" + }, + "default": {} + }, "tasks": { "type": "array", "items": { @@ -686,6 +693,39 @@ "always" ] }, + "SubworkflowDefinition": { + "type": "object", + "properties": { + "version": { + "type": "string" + }, + "inputs": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "inputSchema": true, + "outputs": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "outputSchema": true, + "tasks": { + "type": "array", + "items": { + "$ref": "#/$defs/TaskDefinition" + } + } + }, + "additionalProperties": false, + "required": [ + "version", + "inputSchema", + "outputSchema", + "tasks" + ] + }, "TaskDefinition": { "type": "object", "properties": { diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 0674149..78bfd87 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 36; +const ACCEPTANCE_SCENARIOS: usize = 37; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1553,6 +1553,70 @@ pub fn run(root: &Path) -> Result<()> { let loop_replay_inspect = inspect(&binary, root, &loop_db, loop_replay_id)?; ensure!(array_len(&loop_replay_inspect, "/data/effects")? == 0); + scenario( + 37, + "packaged CLI expands, inspects, and replays a typed sub-workflow", + ); + let subworkflow = root.join("examples/v1/subworkflow.yaml"); + let subworkflow_plan = successful_json( + &binary, + root, + &strings([ + "plan", + path(&subworkflow)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq( + &subworkflow_plan, + "/data/tasks/summary/uses/kind", + "subworkflow_aggregate", + )?; + ensure_eq( + &subworkflow_plan, + "/data/tasks/summary/uses/name/version", + "1.0.0", + )?; + ensure!( + subworkflow_plan + .pointer("/data/tasks") + .and_then(Value::as_object) + .is_some_and(|tasks| tasks.keys().any(|id| id.starts_with("summary--inputs-"))) + ); + let subworkflow_db = directory.path().join("subworkflow.db"); + let subworkflow_run = successful_json( + &binary, + root, + &run_args(&subworkflow, &subworkflow_db, root, &[]), + )?; + ensure_eq(&subworkflow_run, "/data/state", "succeeded")?; + ensure_eq(&subworkflow_run, "/data/output/result", "durable")?; + let subworkflow_run_id = string_at(&subworkflow_run, "/data/runId")?; + let subworkflow_inspect = inspect(&binary, root, &subworkflow_db, subworkflow_run_id)?; + ensure!(array_len(&subworkflow_inspect, "/data/tasks")? == 4); + let subworkflow_replay = successful_json( + &binary, + root, + &strings([ + "replay", + subworkflow_run_id, + "--db", + path(&subworkflow_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&subworkflow_replay, "/data/state", "succeeded")?; + let subworkflow_replay_id = string_at(&subworkflow_replay, "/data/runId")?; + let subworkflow_replay_inspect = + inspect(&binary, root, &subworkflow_db, subworkflow_replay_id)?; + ensure!(array_len(&subworkflow_replay_inspect, "/data/effects")? == 0); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } From 07b0f0827efb8c368e7563ae621a1a5f99639013 Mon Sep 17 00:00:00 2001 From: Ompragash Date: Sat, 25 Jul 2026 13:53:28 +0530 Subject: [PATCH 16/44] feat: add durable compensation workflows --- README.md | 4 +- crates/agentctl-cli/src/main.rs | 152 +- crates/agentctl-core/src/compiler.rs | 437 +++++- crates/agentctl-core/src/dsl.rs | 97 +- crates/agentctl-core/src/tool.rs | 2 - crates/agentctl-providers/src/lib.rs | 1 - crates/agentctl-runtime/src/lib.rs | 1313 ++++++++++++++++- crates/agentctl-store/src/lib.rs | 60 +- docs/ARCHITECTURE.md | 9 +- docs/COMPATIBILITY.md | 6 +- docs/DSL.md | 10 +- docs/DURABLE_EXECUTION.md | 9 + docs/LIMITATIONS.md | 5 +- docs/TOOLS.md | 2 +- ...0013-compensation-as-source-linked-runs.md | 36 + docs/execution/COMPATIBILITY.md | 10 +- docs/execution/COMPLETENESS_VERIFICATION.md | 3 +- docs/execution/DECISIONS.md | 1 + docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 30 +- docs/generated/CLI.md | 1 + docs/guides/COMPENSATION.md | 107 ++ docs/reference/YAML.md | 9 +- examples/v1/README.md | 1 + examples/v1/compensation.yaml | 40 + schemas/workflow.schema.json | 77 +- xtask/src/acceptance.rs | 133 +- 27 files changed, 2464 insertions(+), 92 deletions(-) create mode 100644 docs/adr/0013-compensation-as-source-linked-runs.md create mode 100644 docs/guides/COMPENSATION.md create mode 100644 examples/v1/compensation.yaml diff --git a/README.md b/README.md index 656904d..64aa672 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,11 @@ agentctl retry workflow.yaml SOURCE_RUN_ID --failed --plan agentctl retry workflow.yaml SOURCE_RUN_ID --failed agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task --plan agentctl repair repaired.workflow.yaml SOURCE_RUN_ID --from failed_task +agentctl compensate SOURCE_RUN_ID --plan +agentctl compensate SOURCE_RUN_ID ``` -See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md) and [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. +See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md), [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md), and [Compensate applied effects](docs/guides/COMPENSATION.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). For confidential workflow history, use [Sensitive-state encryption](docs/guides/SENSITIVE_STATE_ENCRYPTION.md). For environment, mounted-file, and policy-gated process credentials, use [Secret references](docs/guides/SECRET_REFERENCES.md). diff --git a/crates/agentctl-cli/src/main.rs b/crates/agentctl-cli/src/main.rs index 07d56bf..a2187e7 100644 --- a/crates/agentctl-cli/src/main.rs +++ b/crates/agentctl-cli/src/main.rs @@ -93,6 +93,8 @@ enum Command { Repair(RepairArgs), /// Retry failed or selected boundaries of an identical terminal workflow. Retry(RetryArgs), + /// Execute explicitly declared best-effort compensation for a terminal run. + Compensate(CompensateArgs), /// Analyze or upgrade retained legacy run records for selective reuse. Runs(RunsArgs), /// Durably request cancellation. @@ -248,6 +250,25 @@ struct RetryArgs { timeout_seconds: Option, } +#[derive(Debug, Args)] +struct CompensateArgs { + source_run_id: String, + #[arg(long = "task")] + task: Vec, + #[arg(long)] + plan: bool, + #[arg(long, default_value = ".agentctl/runtime.db")] + db: PathBuf, + #[arg(long)] + interactive: bool, + #[arg(long)] + diff: bool, + #[arg(long)] + workspace: Option, + #[arg(long)] + timeout_seconds: Option, +} + #[derive(Debug, Args)] struct RunsArgs { #[arg(long, default_value = ".agentctl/runtime.db")] @@ -722,6 +743,7 @@ async fn execute(cli: Cli) -> Result { } Command::Repair(args) => repair_workflow(output, args).await, Command::Retry(args) => retry_workflow(output, args).await, + Command::Compensate(args) => compensate_workflow(output, args).await, Command::Runs(args) => runs_command(output, args), Command::Cancel(args) => { let store = open_store(&args.db)?; @@ -783,7 +805,10 @@ async fn execute(cli: Cli) -> Result { audit.len(), traces.len(), ); - let human = if matches!(run.mode, RunMode::Repair | RunMode::Retry) { + let human = if matches!( + run.mode, + RunMode::Repair | RunMode::Retry | RunMode::Compensation + ) { let reused = tasks .iter() .filter(|task| task.disposition == TaskDisposition::Reused) @@ -1148,6 +1173,103 @@ async fn retry_workflow(output: OutputFormat, args: RetryArgs) -> Result Result { + if !args.plan { + validate_interactive(args.interactive)?; + } + let store = open_store(&args.db)?; + let source = store + .load_run(&args.source_run_id) + .map_err(CliError::persistence)?; + let workflow: Workflow = serde_json::from_value(source.workflow.clone()) + .map_err(|error| CliError::persistence(error.to_string()))?; + let base = resolve_base_path( + args.workspace + .as_deref() + .or_else(|| source.base_path.as_deref().map(Path::new)), + )?; + let planner = Runtime::new(store.clone(), &base); + let plan = planner + .plan_compensation(&args.source_run_id, &args.task) + .map_err(map_runtime_error)?; + if args.plan || !plan.executable { + let human = format!( + "compensation plan: {}\nsource: {}\nexecute: {}\nalready compensated: {}\nblocked: {}", + if plan.complete { + "complete" + } else if plan.executable { + "partial" + } else { + "blocked" + }, + plan.source_run_id, + plan.tasks + .iter() + .map(|task| format!("{}->{}", task.source_task_id, task.compensation_task_id)) + .collect::>() + .join(", "), + plan.already_compensated_effects.join(", "), + plan.blocked + .iter() + .map(|block| format!("{}: {}", block.task_id, block.message)) + .collect::>() + .join("; "), + ); + let complete = plan.complete; + let executable = plan.executable; + print_value(output, "CompensationPlan", &plan, Vec::new(), human)?; + return Ok(if complete || (!executable && plan.blocked.is_empty()) { + EXIT_OK + } else { + EXIT_POLICY + }); + } + let cancellation = cancellation_token(args.timeout_seconds); + let registry = build_registry(&workflow, &base, &cancellation, None).await?; + let runtime = Runtime::new(store, &base).with_registry(registry); + let outcome = runtime + .compensate( + plan, + RunOptions { + check: false, + diff: args.diff, + interactive: args.interactive, + }, + &cancellation, + ) + .await + .map_err(map_runtime_error)?; + let human = format!( + "compensation {} source={} state={} compensated={} failed={} blocked={}", + outcome.run_id.as_deref().unwrap_or("not-created"), + outcome.source_run_id, + outcome + .state + .map_or_else(|| "not-run".to_owned(), |state| format!("{state:?}")), + outcome.compensated_tasks.join(","), + outcome.failed_tasks.join(","), + outcome + .blocked + .iter() + .map(|block| block.task_id.as_str()) + .collect::>() + .join(","), + ); + print_value(output, "CompensationOutcome", &outcome, Vec::new(), human)?; + Ok(match outcome.state { + Some(state) => { + let code = outcome_exit_code(state); + if code == EXIT_OK && !outcome.blocked.is_empty() { + EXIT_POLICY + } else { + code + } + } + None if outcome.blocked.is_empty() => EXIT_OK, + None => EXIT_POLICY, + }) +} + fn print_outcome( output: OutputFormat, outcome: &agentctl_runtime::RunOutcome, @@ -2137,6 +2259,11 @@ fn qualify_pack_task( break; } } + if let Some(compensate) = &mut task.compensate + && let Some(name) = compensate.uses.strip_prefix("action:") + { + compensate.uses = format!("action:{}", qualify(name)); + } } fn insert_pack_item( @@ -2180,7 +2307,7 @@ async fn build_registry( .filter_map(|agent| workflow.spec.agents.get(agent)) .map(|agent| agent.provider.as_str()) .collect::>(); - let selected_action_kinds = workflow + let mut selected_action_kinds = workflow .spec .tasks .iter() @@ -2189,6 +2316,27 @@ async fn build_registry( .filter_map(|action| workflow.spec.actions.get(action)) .map(|action| action.kind) .collect::>(); + selected_action_kinds.extend( + workflow + .spec + .tasks + .iter() + .filter_map(|task| task.compensate.as_ref()) + .filter_map(|compensate| compensate.uses.strip_prefix("action:")) + .filter_map(|action| workflow.spec.actions.get(action)) + .map(|action| action.kind), + ); + selected_action_kinds.extend( + workflow + .spec + .subworkflows + .values() + .flat_map(|definition| definition.tasks.iter()) + .filter_map(|task| task.compensate.as_ref()) + .filter_map(|compensate| compensate.uses.strip_prefix("action:")) + .filter_map(|action| workflow.spec.actions.get(action)) + .map(|action| action.kind), + ); for (name, definition) in &workflow.spec.providers { let credential = definition.credential.clone().unwrap_or_else(|| { SecretReference::environment(default_credential_env(definition.kind.clone())) diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index ab9f3c7..bf4cb4e 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -38,10 +38,21 @@ pub struct CompiledTask { pub retry: RetryDefinition, pub timeout_seconds: u64, pub failure: crate::dsl::FailureBehavior, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compensate: Option, pub output_schema: Option, pub predictability: PlanPredictability, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompiledCompensation { + pub uses: String, + pub input: JsonMap, + pub retry: RetryDefinition, + pub timeout_seconds: u64, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CompiledExpansion { @@ -363,6 +374,7 @@ fn expand_tasks( aggregate.input.clear(); aggregate.retry = RetryDefinition::default(); aggregate.timeout_seconds = None; + aggregate.compensate = None; aggregate.output_schema = None; expanded.push(ExpandedTaskDefinition { definition: aggregate, @@ -587,6 +599,7 @@ fn expand_tasks( aggregate.input.clear(); aggregate.retry = RetryDefinition::default(); aggregate.timeout_seconds = None; + aggregate.compensate = None; aggregate.output_schema = None; expanded.push(ExpandedTaskDefinition { definition: aggregate, @@ -701,6 +714,7 @@ fn instantiate_subworkflow_task( || !task.memory_writes.is_empty() || task.retry != RetryDefinition::default() || task.timeout_seconds.is_some() + || task.compensate.is_some() || task.output_schema.is_some() { diagnostics.push(Diagnostic::error( @@ -823,6 +837,7 @@ fn synthetic_task(source: &TaskDefinition, id: String, needs: Vec) -> Ta task.when = None; task.vars.clear(); task.input.clear(); + task.compensate = None; task.retry = RetryDefinition::default(); task.timeout_seconds = None; task.output_schema = None; @@ -836,7 +851,39 @@ fn namespace_subworkflow_action_state( file: &str, diagnostics: &mut Vec, ) { - let Some(name) = task.uses.strip_prefix("action:") else { + namespace_subworkflow_action_input( + workflow, + &task.id, + &task.uses, + &mut task.input, + prefix, + file, + diagnostics, + ); + if let Some(compensate) = &mut task.compensate { + namespace_subworkflow_action_input( + workflow, + &task.id, + &compensate.uses, + &mut compensate.input, + prefix, + file, + diagnostics, + ); + } +} + +#[allow(clippy::too_many_arguments)] +fn namespace_subworkflow_action_input( + workflow: &Workflow, + task_id: &str, + uses: &str, + input: &mut JsonMap, + prefix: &str, + file: &str, + diagnostics: &mut Vec, +) { + let Some(name) = uses.strip_prefix("action:") else { return; }; let Some(action) = workflow.spec.actions.get(name) else { @@ -847,17 +894,14 @@ fn namespace_subworkflow_action_state( ActionKind::LongTermMemoryRead | ActionKind::LongTermMemoryWrite => "namespace", _ => return, }; - let Some(Value::String(value)) = task.input.get_mut(field) else { + let Some(Value::String(value)) = input.get_mut(field) else { return; }; if value.contains("${{") { diagnostics.push(Diagnostic::error( DiagnosticCode::UnsupportedCapability, file, - format!( - "sub-workflow task `{}` requires a static `{field}` for isolated state", - task.id - ), + format!("sub-workflow task `{task_id}` requires a static `{field}` for isolated state"), )); } else { value.insert_str(0, prefix); @@ -876,6 +920,10 @@ fn rewrite_subworkflow_task( .map(|value| rewrite_subworkflow_string(value, local_ids, input_id, memory_prefix)); task.vars = rewrite_subworkflow_map(&task.vars, local_ids, input_id, memory_prefix); task.input = rewrite_subworkflow_map(&task.input, local_ids, input_id, memory_prefix); + if let Some(compensate) = &mut task.compensate { + compensate.input = + rewrite_subworkflow_map(&compensate.input, local_ids, input_id, memory_prefix); + } task.memory_writes = task .memory_writes .iter() @@ -1135,6 +1183,8 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result JsonMap::new(), }; vars.extend(task.vars.clone()); + let compensate = + compile_compensation(&workflow, task, &task_use, file, position, &mut diagnostics); declaration_order.push(task.id.clone()); source_positions.insert(task.id.clone(), position); tasks.insert( @@ -1154,6 +1204,7 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result Result { - let mut effects = vec![EffectRequirement { - task: task.id.clone(), - operation: format!("agent:{agent_name}"), - effect_class: EffectClass::Model, - approval_possible: workflow.spec.policy.approval - != crate::dsl::ApprovalMode::Never, - predictability: task.predictability, - }]; - if let Some(agent) = workflow.spec.agents.get(agent_name) { - effects.extend(agent.tools.iter().filter_map(|name| { - workflow.spec.tools.get(name).map(|tool| EffectRequirement { - task: task.id.clone(), - operation: format!("tool:{name}"), - effect_class: tool.effect_class, - approval_possible: tool.approval - != crate::dsl::ApprovalRequirement::Never - || workflow.spec.policy.approval != crate::dsl::ApprovalMode::Never, - predictability: PlanPredictability::RequiresExecution, - }) - })); + .flat_map(|task| { + let mut effects = match &task.uses { + TaskUse::Agent(agent_name) => { + let mut effects = vec![EffectRequirement { + task: task.id.clone(), + operation: format!("agent:{agent_name}"), + effect_class: EffectClass::Model, + approval_possible: workflow.spec.policy.approval + != crate::dsl::ApprovalMode::Never, + predictability: task.predictability, + }]; + if let Some(agent) = workflow.spec.agents.get(agent_name) { + effects.extend(agent.tools.iter().filter_map(|name| { + workflow.spec.tools.get(name).map(|tool| EffectRequirement { + task: task.id.clone(), + operation: format!("tool:{name}"), + effect_class: tool.effect_class, + approval_possible: tool.approval + != crate::dsl::ApprovalRequirement::Never + || workflow.spec.policy.approval + != crate::dsl::ApprovalMode::Never, + predictability: PlanPredictability::RequiresExecution, + }) + })); + } + effects } - effects - } - TaskUse::Action(name) => { + TaskUse::Action(name) => { + let effect_class = workflow + .spec + .actions + .get(name) + .map_or(EffectClass::ExternalMutate, |action| { + action_effect_class(action.kind) + }); + vec![EffectRequirement { + task: task.id.clone(), + operation: format!("action:{name}"), + effect_class, + approval_possible: workflow.spec.policy.approval + != crate::dsl::ApprovalMode::Never, + predictability: task.predictability, + }] + } + TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => Vec::new(), + }; + if let Some(compensate) = &task.compensate { + let name = compensate.uses.trim_start_matches("action:"); let effect_class = workflow .spec .actions @@ -1680,20 +1758,16 @@ fn plan_requirements( .map_or(EffectClass::ExternalMutate, |action| { action_effect_class(action.kind) }); - vec![EffectRequirement { + effects.push(EffectRequirement { task: task.id.clone(), - operation: format!("action:{name}"), + operation: format!("compensate:{}", compensate.uses), effect_class, approval_possible: workflow.spec.policy.approval != crate::dsl::ApprovalMode::Never, - predictability: task.predictability, - }] + predictability: PlanPredictability::RequiresExecution, + }); } - TaskUse::Aggregate(_) - | TaskUse::Router(_) - | TaskUse::LoopAggregate(_) - | TaskUse::SubworkflowInput(_) - | TaskUse::SubworkflowAggregate(_) => Vec::new(), + effects }) .collect(); PlanRequirements { @@ -1717,6 +1791,149 @@ const fn action_effect_class(kind: ActionKind) -> EffectClass { } } +fn compile_compensation( + workflow: &Workflow, + task: &TaskDefinition, + task_use: &TaskUse, + file: &str, + position: usize, + diagnostics: &mut Vec, +) -> Option { + let definition = task.compensate.as_ref()?; + let path = format!("spec.tasks[{position}].compensate"); + if !task_use_supports_compensation(workflow, task_use) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::UnsupportedCapability, + file, + format!( + "task `{}` declares compensation but has no potentially mutating effect", + task.id + ), + ) + .with_path(path.clone()), + ); + } + let Some(action_name) = definition.uses.strip_prefix("action:") else { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::UnsupportedCapability, + file, + format!("task `{}` compensation must use a named action", task.id), + ) + .with_path(format!("{path}.uses")), + ); + return None; + }; + let Some(action) = workflow.spec.actions.get(action_name) else { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::MissingReference, + file, + format!( + "task `{}` compensation refers to unknown action `{action_name}`", + task.id + ), + ) + .with_path(format!("{path}.uses")), + ); + return None; + }; + if !action_kind_supports_compensation(action.kind) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::UnsupportedCapability, + file, + format!( + "task `{}` compensation action `{action_name}` is not effectful", + task.id + ), + ) + .with_path(format!("{path}.uses")), + ); + } + if definition.retry.max_attempts == 0 || definition.retry.max_attempts > 20 { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "compensate.retry.maxAttempts must be between 1 and 20", + ) + .with_path(format!("{path}.retry.maxAttempts")), + ); + } + if definition.retry.backoff_ms > 60_000 { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "compensate.retry.backoffMs must not exceed 60000", + ) + .with_path(format!("{path}.retry.backoffMs")), + ); + } + if definition + .timeout_seconds + .is_some_and(|value| value == 0 || value > 86_400) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "compensate.timeoutSeconds must be between 1 and 86400", + ) + .with_path(format!("{path}.timeoutSeconds")), + ); + } + let mut input = action.defaults.clone(); + input.extend(definition.input.clone()); + Some(CompiledCompensation { + uses: definition.uses.clone(), + input, + retry: definition.retry.clone(), + timeout_seconds: definition + .timeout_seconds + .unwrap_or(workflow.spec.runtime.default_timeout_seconds), + }) +} + +fn task_use_supports_compensation(workflow: &Workflow, task_use: &TaskUse) -> bool { + match task_use { + TaskUse::Action(name) => workflow + .spec + .actions + .get(name) + .is_some_and(|action| action_kind_supports_compensation(action.kind)), + TaskUse::Agent(name) => workflow.spec.agents.get(name).is_some_and(|agent| { + agent.tools.iter().any(|tool| { + workflow.spec.tools.get(tool).is_some_and(|definition| { + !matches!( + definition.effect_class, + EffectClass::Pure | EffectClass::Observe | EffectClass::Model + ) + }) + }) + }), + TaskUse::Aggregate(_) + | TaskUse::Router(_) + | TaskUse::LoopAggregate(_) + | TaskUse::SubworkflowInput(_) + | TaskUse::SubworkflowAggregate(_) => false, + } +} + +const fn action_kind_supports_compensation(kind: ActionKind) -> bool { + matches!( + kind, + ActionKind::Write + | ActionKind::ShellExec + | ActionKind::MemoryWrite + | ActionKind::LongTermMemoryWrite + | ActionKind::McpCall + | ActionKind::A2aDelegate + ) +} + fn validate_task_templates( task: &CompiledTask, tasks: &BTreeMap, @@ -1771,6 +1988,53 @@ fn validate_task_templates( } } +fn validate_compensation_templates( + task: &CompiledTask, + tasks: &BTreeMap, + file: &str, + position: usize, + diagnostics: &mut Vec, +) { + let Some(compensate) = &task.compensate else { + return; + }; + let mut values = compensate.input.values().collect::>(); + while let Some(value) = values.pop() { + match value { + Value::String(template) => { + if let Err(error) = validate_expression(template) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::InvalidTemplate, + file, + format!("task `{}` compensation: {error}", task.id), + ) + .with_path(format!("spec.tasks[{position}].compensate.with")), + ); + } + for reference in referenced_tasks(template) { + if !tasks.contains_key(&reference) { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::MissingReference, + file, + format!( + "task `{}` compensation refers to unknown task `{reference}`", + task.id + ), + ) + .with_path(format!("spec.tasks[{position}].compensate.with")), + ); + } + } + } + Value::Array(items) => values.extend(items), + Value::Object(map) => values.extend(map.values()), + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } + } +} + fn push_template_error( error: TemplateError, task: &CompiledTask, @@ -2796,6 +3060,99 @@ spec: assert_eq!(plan.tasks["right--write"].input["key"], "right__result"); } + #[test] + fn compensation_is_explicit_effectful_and_part_of_the_plan() { + let workflow = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: compensation } +spec: + compensation: { onFailure: automatic } + actions: + write: { kind: builtin.write } + tasks: + - id: create + uses: action:write + with: { path: resource.txt, content: created } + compensate: + uses: action:write + with: + path: "${{ tasks.create.output.path }}" + content: removed + retry: { maxAttempts: 2, backoffMs: 5 } +"#, + ); + let plan = compile(&workflow, "fixture.yaml").expect("compensation compiles"); + let compensate = plan.tasks["create"] + .compensate + .as_ref() + .expect("compiled compensation"); + assert_eq!(compensate.uses, "action:write"); + assert_eq!(compensate.retry.max_attempts, 2); + assert!( + plan.requirements + .effects + .iter() + .any(|effect| effect.operation == "compensate:action:write") + ); + + let invalid = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: invalid-compensation } +spec: + actions: + assign: { kind: builtin.assign } + read: { kind: builtin.read } + tasks: + - id: pure + uses: action:assign + compensate: { uses: "action:read" } +"#, + ); + let diagnostics = compile(&invalid, "fixture.yaml").expect_err("invalid compensation"); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("has no potentially mutating effect") + })); + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("compensation action `read` is not effectful") + })); + + let expanded = parse( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: expanded-compensation } +spec: + actions: + write: { kind: builtin.write } + tasks: + - id: create + uses: action:write + foreach: { items: [one, two], as: item } + with: { path: "${{ vars.item }}", content: created } + compensate: + uses: action:write + with: { path: "${{ vars.item }}", content: removed } +"#, + ); + let expanded = compile(&expanded, "fixture.yaml").expect("expanded compensation"); + assert!(expanded.tasks["create"].compensate.is_none()); + assert!( + expanded + .tasks + .iter() + .filter(|(id, _)| id.starts_with("create--")) + .all(|(_, task)| task.compensate.is_some()) + ); + } + #[test] fn typed_router_cases_compile_to_explicit_destination_guards() { let workflow = parse( diff --git a/crates/agentctl-core/src/dsl.rs b/crates/agentctl-core/src/dsl.rs index 1ebf0f8..2150a20 100644 --- a/crates/agentctl-core/src/dsl.rs +++ b/crates/agentctl-core/src/dsl.rs @@ -52,6 +52,8 @@ pub struct WorkflowSpec { pub tools: BTreeMap, #[serde(default)] pub subworkflows: BTreeMap, + #[serde(default)] + pub compensation: CompensationPolicyDefinition, pub tasks: Vec, #[serde(default)] pub policy: PolicyDefinition, @@ -359,8 +361,6 @@ pub struct ToolDefinition { pub network: Vec, #[serde(default)] pub approval: ApprovalRequirement, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub compensation: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -445,9 +445,40 @@ pub struct TaskDefinition { #[serde(default)] pub failure: FailureBehavior, #[serde(default, skip_serializing_if = "Option::is_none")] + pub compensate: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub output_schema: Option, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CompensationDefinition { + pub uses: String, + #[serde(default, rename = "with")] + pub input: JsonMap, + #[serde(default)] + pub retry: RetryDefinition, + #[serde(default)] + pub timeout_seconds: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CompensationTrigger { + #[default] + Manual, + Automatic, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CompensationPolicyDefinition { + #[serde(default)] + pub on_failure: CompensationTrigger, + #[serde(default)] + pub approval: ApprovalRequirement, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SubworkflowDefinition { @@ -1171,6 +1202,43 @@ fn validate_document(workflow: &Workflow, file: &str) -> Vec { .with_path(format!("spec.tasks[{position}].retry.backoffMs")), ); } + if let Some(compensate) = &task.compensate { + if compensate + .timeout_seconds + .is_some_and(|value| value == 0 || value > MAX_PROCESS_TIMEOUT_SECONDS) + { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "compensate.timeoutSeconds must be between 1 and 86400", + ) + .with_path(format!("spec.tasks[{position}].compensate.timeoutSeconds")), + ); + } + if compensate.retry.max_attempts == 0 || compensate.retry.max_attempts > 20 { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "compensate.retry.maxAttempts must be between 1 and 20", + ) + .with_path(format!( + "spec.tasks[{position}].compensate.retry.maxAttempts" + )), + ); + } + if compensate.retry.backoff_ms > 60_000 { + diagnostics.push( + Diagnostic::error( + DiagnosticCode::SchemaViolation, + file, + "compensate.retry.backoffMs must not exceed 60000", + ) + .with_path(format!("spec.tasks[{position}].compensate.retry.backoffMs")), + ); + } + } } for (name, server) in &workflow.spec.mcp_servers { for (header, secret) in &server.headers { @@ -1321,6 +1389,31 @@ spec: ); } + #[test] + fn rejects_removed_tool_level_compensation_metadata() { + let source = MINIMAL.replace( + " actions:", + r#" tools: + legacy: + kind: builtin.echo + description: echo + inputSchema: { type: object } + outputSchema: { type: object } + capability: observe + risk: low + effectClass: pure + idempotency: pure + retrySafe: true + timeoutSeconds: 5 + compensation: undo + actions:"#, + ); + let diagnostics = + parse_workflow(&source, "legacy-compensation.yaml").expect_err("removed field"); + assert_eq!(diagnostics[0].code, DiagnosticCode::SchemaViolation); + assert!(diagnostics[0].message.contains("compensation")); + } + #[test] fn validates_secret_reference_name() { let source = MINIMAL.replace( diff --git a/crates/agentctl-core/src/tool.rs b/crates/agentctl-core/src/tool.rs index 117dc96..eada89c 100644 --- a/crates/agentctl-core/src/tool.rs +++ b/crates/agentctl-core/src/tool.rs @@ -24,7 +24,6 @@ pub struct ToolContract { pub network_requirements: Vec, pub approval: ApprovalRequirement, pub observability: Value, - pub compensation: Option, } impl ToolContract { @@ -109,7 +108,6 @@ mod tests { network_requirements: Vec::new(), approval: ApprovalRequirement::Never, observability: Value::Null, - compensation: None, } } diff --git a/crates/agentctl-providers/src/lib.rs b/crates/agentctl-providers/src/lib.rs index ce79ba5..6854035 100644 --- a/crates/agentctl-providers/src/lib.rs +++ b/crates/agentctl-providers/src/lib.rs @@ -1219,7 +1219,6 @@ mod tests { network_requirements: Vec::new(), approval: ApprovalRequirement::Never, observability: Value::Null, - compensation: None, }], max_output_tokens: 64, reasoning: None, diff --git a/crates/agentctl-runtime/src/lib.rs b/crates/agentctl-runtime/src/lib.rs index fb005a5..d620f02 100644 --- a/crates/agentctl-runtime/src/lib.rs +++ b/crates/agentctl-runtime/src/lib.rs @@ -5,10 +5,11 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; -use agentctl_core::compiler::{CompiledPlan, PlanPredictability, TaskUse}; +use agentctl_core::compiler::{CompiledPlan, PlanPredictability, TaskUse, compile}; use agentctl_core::dsl::{ - API_VERSION, ActionDefinition, ActionKind, ApprovalRequirement, EffectClass, FailureBehavior, - Idempotency, Risk, ToolDefinition, ToolKind, Workflow, + API_VERSION, ActionDefinition, ActionKind, ApprovalRequirement, CompensationTrigger, + EffectClass, FailureBehavior, Idempotency, Risk, TaskDefinition, ToolDefinition, ToolKind, + Workflow, }; use agentctl_core::effect::{ ActionResult, ChangeStatus, EffectRecord, EffectRequest, EffectStatus, @@ -118,7 +119,6 @@ impl BuiltinToolExecutor { network_requirements: definition.network.clone(), approval: definition.approval, observability: Value::Null, - compensation: definition.compensation.clone(), }, kind: definition.kind, policy, @@ -373,6 +373,56 @@ pub struct RetryOutcome { pub output: Option, } +pub const COMPENSATION_PLAN_VERSION: &str = "agentctl.dev/compensation-plan/v1"; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompensationBlock { + pub task_id: String, + pub effect_id: Option, + pub rule: String, + pub message: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CompensationTaskPlan { + pub source_task_id: String, + pub compensation_task_id: String, + pub action: String, + pub source_effect_ids: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CompensationPlan { + pub api_version: String, + pub source_run_id: String, + pub executable: bool, + pub complete: bool, + pub automatic: bool, + pub selected_tasks: Vec, + pub tasks: Vec, + pub blocked: Vec, + pub already_compensated_effects: Vec, + #[serde(skip_serializing)] + workflow: Workflow, + #[serde(skip_serializing)] + compiled: CompiledPlan, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompensationOutcome { + pub run_id: Option, + pub source_run_id: String, + pub trace_id: Option, + pub state: Option, + pub compensated_tasks: Vec, + pub failed_tasks: Vec, + pub blocked: Vec, +} + pub const LEGACY_UPGRADE_ANALYSIS_VERSION: &str = "agentctl.dev/legacy-upgrade/v1"; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -542,6 +592,7 @@ impl Runtime { &Value::Object(workflow.spec.memory.working.clone().into_iter().collect()), mode, None, + None, &self.base_path, self.clock.now(), &trace_id, @@ -655,7 +706,11 @@ impl Runtime { &trace_id, )?; } - self.drive(run_id, &trace_id, options, cancellation).await + let outcome = self.drive(run_id, &trace_id, options, cancellation).await?; + if run.mode == RunMode::Compensation { + self.finalize_compensation_run(run_id)?; + } + Ok(outcome) } fn prepare_reconciled_resume(&self, run_id: &str, trace_id: &str) -> Result<(), RuntimeError> { @@ -769,6 +824,7 @@ impl Runtime { &source.working_memory, RunMode::Replay, Some(source_run_id), + None, Path::new(source.base_path.as_deref().unwrap_or(".")), self.clock.now(), &trace_id, @@ -1222,6 +1278,7 @@ impl Runtime { &serde_json::to_value(&workflow.spec.memory.working)?, RunMode::Fork, Some(source_run_id), + None, &self.base_path, self.clock.now(), &trace_id, @@ -2210,6 +2267,512 @@ impl Runtime { }) } + pub fn plan_compensation( + &self, + source_run_id: &str, + selected_tasks: &[String], + ) -> Result { + let source = self.store.load_run(source_run_id)?; + if !source.state.is_terminal() { + return Err(RuntimeError::InvalidState(format!( + "compensation source run `{source_run_id}` is not terminal ({:?})", + source.state + ))); + } + let compensation_runs = self.store.compensation_runs(source_run_id)?; + for (run_id, state) in &compensation_runs { + if state.is_terminal() { + self.finalize_compensation_run(run_id)?; + } + } + + let source_workflow: Workflow = serde_json::from_value(source.workflow.clone())?; + let automatic = + source_workflow.spec.compensation.on_failure == CompensationTrigger::Automatic; + let source_tasks = self.store.list_tasks(source_run_id)?; + let source_effects = self.store.list_effects(source_run_id)?; + let selected = selected_tasks.iter().cloned().collect::>(); + let mut blocked = Vec::new(); + for task_id in &selected { + match source.plan.tasks.get(task_id) { + None => blocked.push(CompensationBlock { + task_id: task_id.clone(), + effect_id: None, + rule: "unknown_task".to_owned(), + message: format!("source run has no task `{task_id}`"), + }), + Some(task) if task.compensate.is_none() => blocked.push(CompensationBlock { + task_id: task_id.clone(), + effect_id: None, + rule: "undeclared_compensation".to_owned(), + message: format!("task `{task_id}` has no compensation declaration"), + }), + Some(_) => {} + } + } + let mut blocked_source_tasks = BTreeSet::new(); + for (run_id, state) in &compensation_runs { + if !state.is_terminal() { + blocked.push(CompensationBlock { + task_id: "*".to_owned(), + effect_id: None, + rule: "compensation_in_progress".to_owned(), + message: format!( + "compensation run `{run_id}` is {state:?}; resume or reconcile it before starting another" + ), + }); + continue; + } + let prior = self.store.load_run(run_id)?; + let workflow: Workflow = serde_json::from_value(prior.workflow)?; + let mappings = workflow + .spec + .inputs + .get("__agentctlCompensation") + .and_then(Value::as_object) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "compensation run `{run_id}` has no durable source mapping" + )) + })?; + for (compensation_task_id, mapping) in mappings { + let Some(effect) = self + .store + .latest_effect_for_task(run_id, compensation_task_id)? + else { + continue; + }; + let reconciliation = self + .store + .latest_effect_reconciliation(&effect.request.id)?; + let rule = match reconciliation.as_ref().map(|record| record.status) { + Some(ReconciliationStatus::Applied | ReconciliationStatus::NotApplied) => None, + Some(ReconciliationStatus::Compensated) => Some("compensation_effect_reversed"), + None if matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) => + { + Some("unreconciled_compensation_effect") + } + None if effect.status == EffectStatus::Succeeded && !effect.confirmed => { + Some("unconfirmed_compensation_effect") + } + None => None, + }; + let Some(rule) = rule else { + continue; + }; + let source_task_id = mapping + .get("sourceTaskId") + .and_then(Value::as_str) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "compensation task `{compensation_task_id}` has no source task mapping" + )) + })?; + blocked_source_tasks.insert(source_task_id.to_owned()); + blocked.push(CompensationBlock { + task_id: source_task_id.to_owned(), + effect_id: Some(effect.request.id.clone()), + rule: rule.to_owned(), + message: format!( + "compensation effect `{}` must be reconciled before task `{source_task_id}` can be compensated again", + effect.request.id + ), + }); + } + } + + let mut generated_tasks = Vec::new(); + let mut task_plans = Vec::new(); + let mut mappings = serde_json::Map::new(); + let mut already_compensated_effects = Vec::new(); + for task_id in source.plan.order.iter().rev() { + if !selected.is_empty() && !selected.contains(task_id) { + continue; + } + let Some(source_task) = source.plan.tasks.get(task_id) else { + continue; + }; + let Some(compensate) = &source_task.compensate else { + continue; + }; + if blocked_source_tasks.contains(task_id) { + continue; + } + let mut applied_effects = Vec::new(); + for effect in source_effects + .iter() + .filter(|effect| effect.request.task_id == *task_id) + .filter(|effect| compensation_relevant_effect(effect.request.effect_class)) + { + let reconciliation = self + .store + .latest_effect_reconciliation(&effect.request.id)?; + match reconciliation.as_ref().map(|record| record.status) { + Some(ReconciliationStatus::Compensated) => { + already_compensated_effects.push(effect.request.id.clone()); + } + Some(ReconciliationStatus::NotApplied) => {} + Some(ReconciliationStatus::Applied) => { + applied_effects.push(effect.request.id.clone()); + } + None if matches!( + effect.status, + EffectStatus::Started | EffectStatus::Uncertain + ) => + { + blocked.push(CompensationBlock { + task_id: task_id.clone(), + effect_id: Some(effect.request.id.clone()), + rule: "unreconciled_source_effect".to_owned(), + message: format!( + "effect `{}` must be reconciled as applied or not-applied before compensation", + effect.request.id + ), + }); + } + None if effect.status == EffectStatus::Succeeded && effect.confirmed => { + applied_effects.push(effect.request.id.clone()); + } + None if effect.status == EffectStatus::Succeeded => { + blocked.push(CompensationBlock { + task_id: task_id.clone(), + effect_id: Some(effect.request.id.clone()), + rule: "unconfirmed_source_effect".to_owned(), + message: format!( + "effect `{}` succeeded without confirmed application", + effect.request.id + ), + }); + } + None => {} + } + } + if applied_effects.is_empty() { + continue; + } + let context = context_for_task(&source, &source_tasks, source_task)?; + let rendered = match render(&serde_json::to_value(&compensate.input)?, &context) { + Ok(rendered) => rendered, + Err(error) => { + blocked.push(CompensationBlock { + task_id: task_id.clone(), + effect_id: None, + rule: "compensation_input_unavailable".to_owned(), + message: format!( + "task `{task_id}` compensation input cannot be reconstructed: {error}" + ), + }); + continue; + } + }; + let input = rendered.as_object().cloned().ok_or_else(|| { + RuntimeError::InvalidState(format!( + "task `{task_id}` compensation input is not an object" + )) + })?; + let compensation_task_id = format!("compensate--{task_id}"); + generated_tasks.push(TaskDefinition { + id: compensation_task_id.clone(), + uses: compensate.uses.clone(), + needs: Vec::new(), + foreach: None, + matrix: None, + route: None, + loop_definition: None, + memory_writes: Vec::new(), + when: None, + vars: BTreeMap::new(), + input: input.into_iter().collect(), + retry: compensate.retry.clone(), + timeout_seconds: Some(compensate.timeout_seconds), + failure: FailureBehavior::Continue, + compensate: None, + output_schema: None, + }); + mappings.insert( + compensation_task_id.clone(), + serde_json::json!({ + "sourceTaskId": task_id, + "sourceEffectIds": applied_effects, + }), + ); + task_plans.push(CompensationTaskPlan { + source_task_id: task_id.clone(), + compensation_task_id, + action: compensate.uses.clone(), + source_effect_ids: applied_effects, + }); + } + + let mut workflow = source_workflow; + workflow.metadata.name = format!("{}-compensation", workflow.metadata.name); + workflow.metadata.labels.insert( + "agentctl.dev/compensation-source".to_owned(), + source_run_id.to_owned(), + ); + workflow.spec.tasks = generated_tasks; + workflow.spec.outputs.clear(); + workflow.spec.inputs = + BTreeMap::from([("__agentctlCompensation".to_owned(), Value::Object(mappings))]); + workflow.spec.runtime.max_concurrency = 1; + match workflow.spec.compensation.approval { + ApprovalRequirement::Policy => {} + ApprovalRequirement::Never => { + workflow.spec.policy.approval = agentctl_core::dsl::ApprovalMode::Never; + } + ApprovalRequirement::Always => { + workflow.spec.policy.approval = agentctl_core::dsl::ApprovalMode::Always; + } + } + workflow.spec.compensation.on_failure = CompensationTrigger::Manual; + workflow.spec.memory.working = source + .working_memory + .as_object() + .cloned() + .unwrap_or_default() + .into_iter() + .collect(); + let compiled = if workflow.spec.tasks.is_empty() { + CompiledPlan { + format_version: source.plan.format_version, + workflow_name: workflow.metadata.name.clone(), + workflow_digest: String::new(), + plan_digest: String::new(), + order: Vec::new(), + max_concurrency: 1, + tasks: BTreeMap::new(), + predictability: PlanPredictability::FullyPredictable, + requirements: agentctl_core::compiler::PlanRequirements { + providers: Vec::new(), + tools: Vec::new(), + effects: Vec::new(), + }, + } + } else { + compile(&workflow, "").map_err(|diagnostics| { + RuntimeError::InvalidState(format!( + "generated compensation workflow is invalid: {}", + diagnostics + .iter() + .map(|diagnostic| diagnostic.message.as_str()) + .collect::>() + .join("; ") + )) + })? + }; + let executable = !task_plans.is_empty() + && !blocked + .iter() + .any(|block| block.rule == "compensation_in_progress"); + Ok(CompensationPlan { + api_version: COMPENSATION_PLAN_VERSION.to_owned(), + source_run_id: source_run_id.to_owned(), + executable, + complete: blocked.is_empty(), + automatic, + selected_tasks: selected.into_iter().collect(), + tasks: task_plans, + blocked, + already_compensated_effects, + workflow, + compiled, + }) + } + + pub async fn compensate( + &self, + plan: CompensationPlan, + options: RunOptions, + cancellation: &CancellationToken, + ) -> Result { + let plan = self.plan_compensation(&plan.source_run_id, &plan.selected_tasks)?; + if !plan.executable { + return Ok(CompensationOutcome { + run_id: None, + source_run_id: plan.source_run_id, + trace_id: None, + state: None, + compensated_tasks: Vec::new(), + failed_tasks: Vec::new(), + blocked: plan.blocked, + }); + } + let source = self.store.load_run(&plan.source_run_id)?; + let run_id = self.ids.next_id("compensation"); + let trace_id = self.ids.next_id("trace"); + let inputs = Value::Object(plan.workflow.spec.inputs.clone().into_iter().collect()); + self.store.create_run( + &run_id, + API_VERSION, + &serde_json::to_value(&plan.workflow)?, + &plan.compiled, + &inputs, + &source.working_memory, + RunMode::Compensation, + None, + Some(&plan.source_run_id), + &self.base_path, + self.clock.now(), + &trace_id, + )?; + self.trace( + TraceEvent::new( + SpanKind::Run, + TracePhase::Started, + "run.compensate", + &trace_id, + &run_id, + self.clock.now(), + ) + .attributes( + serde_json::json!({ + "sourceRunId": plan.source_run_id, + "tasks": plan.tasks, + "blocked": plan.blocked, + }), + &[], + ), + )?; + let result = self + .drive_inner(&run_id, &trace_id, options, cancellation) + .await; + self.finalize_compensation_run(&run_id)?; + let outcome = result?; + let records = self.store.list_tasks(&run_id)?; + let source_by_task = plan + .tasks + .iter() + .map(|task| { + ( + task.compensation_task_id.as_str(), + task.source_task_id.clone(), + ) + }) + .collect::>(); + let compensated_tasks = records + .iter() + .filter(|task| task.state == TaskState::Succeeded) + .filter_map(|task| source_by_task.get(task.task_id.as_str()).cloned()) + .collect(); + let failed_tasks = records + .iter() + .filter(|task| task.state == TaskState::Failed) + .filter_map(|task| source_by_task.get(task.task_id.as_str()).cloned()) + .collect(); + Ok(CompensationOutcome { + run_id: Some(run_id), + source_run_id: plan.source_run_id, + trace_id: Some(outcome.trace_id), + state: Some(outcome.state), + compensated_tasks, + failed_tasks, + blocked: plan.blocked, + }) + } + + fn finalize_compensation_run(&self, run_id: &str) -> Result<(), RuntimeError> { + let run = self.store.load_run(run_id)?; + if run.mode != RunMode::Compensation { + return Ok(()); + } + let source_run_id = run.source_run_id.as_deref().ok_or_else(|| { + RuntimeError::InvalidState(format!("compensation run `{run_id}` has no source run")) + })?; + let workflow: Workflow = serde_json::from_value(run.workflow)?; + let mappings = workflow + .spec + .inputs + .get("__agentctlCompensation") + .and_then(Value::as_object) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "compensation run `{run_id}` has no durable source mapping" + )) + })?; + let tasks = self + .store + .list_tasks(run_id)? + .into_iter() + .map(|task| (task.task_id.clone(), task)) + .collect::>(); + for (compensation_task_id, mapping) in mappings { + let task = tasks.get(compensation_task_id); + let Some(compensation_effect) = self + .store + .latest_effect_for_task(run_id, compensation_task_id)? + else { + if task.is_some_and(|task| task.state == TaskState::Succeeded) { + return Err(RuntimeError::InvalidState(format!( + "successful compensation task `{compensation_task_id}` has no effect" + ))); + } + continue; + }; + let compensation_reconciliation = self + .store + .latest_effect_reconciliation(&compensation_effect.request.id)?; + let compensation_applied = compensation_effect.status == EffectStatus::Succeeded + && compensation_effect.confirmed + || compensation_reconciliation + .as_ref() + .is_some_and(|record| record.status == ReconciliationStatus::Applied); + if !compensation_applied { + continue; + } + let source_effect_ids = mapping + .get("sourceEffectIds") + .and_then(Value::as_array) + .ok_or_else(|| { + RuntimeError::InvalidState(format!( + "compensation task `{compensation_task_id}` has no source effect mapping" + )) + })?; + for source_effect_id in source_effect_ids { + let source_effect_id = source_effect_id.as_str().ok_or_else(|| { + RuntimeError::InvalidState( + "compensation source effect ID is not a string".to_owned(), + ) + })?; + if self + .store + .latest_effect_reconciliation(source_effect_id)? + .as_ref() + .is_some_and(|record| record.status == ReconciliationStatus::Compensated) + { + continue; + } + self.store.reconcile_effect( + &EffectReconciliationRequest { + reconciliation_id: self.ids.next_id("reconciliation"), + effect_id: source_effect_id.to_owned(), + status: ReconciliationStatus::Compensated, + actor: "agentctl-runtime".to_owned(), + reason: "declared compensation completed".to_owned(), + evidence: serde_json::json!({ + "source": "declared-compensation", + "sourceRunId": source_run_id, + "compensationRunId": run_id, + "compensationTaskId": compensation_task_id, + }), + result: None, + result_schema: None, + authorization: serde_json::json!({ + "kind": "workflow-declaration", + "sourceRunId": source_run_id, + }), + compensation_effect_id: Some(compensation_effect.request.id.clone()), + trace_id: self.ids.next_id("trace"), + }, + self.clock.now(), + )?; + } + } + Ok(()) + } + #[allow(clippy::too_many_arguments)] pub async fn repair( &self, @@ -2337,6 +2900,31 @@ impl Runtime { trace_id: &str, options: RunOptions, cancellation: &CancellationToken, + ) -> Result { + let run = self.store.load_run(run_id)?; + let workflow: Workflow = serde_json::from_value(run.workflow)?; + let automatic = run.mode != RunMode::Compensation + && workflow.spec.compensation.on_failure == CompensationTrigger::Automatic; + let result = self + .drive_inner(run_id, trace_id, options, cancellation) + .await; + let failed = matches!(&result, Ok(outcome) if outcome.state == RunState::Failed) + || matches!(&result, Err(RuntimeError::RunFailed { .. })); + if automatic && failed { + let plan = self.plan_compensation(run_id, &[])?; + if plan.executable { + self.compensate(plan, options, cancellation).await?; + } + } + result + } + + async fn drive_inner( + &self, + run_id: &str, + trace_id: &str, + options: RunOptions, + cancellation: &CancellationToken, ) -> Result { let run = self.store.load_run(run_id)?; let workflow: Workflow = serde_json::from_value(run.workflow)?; @@ -3722,7 +4310,7 @@ impl Runtime { "path": path, "predictability": "fully_predictable", }); - if options.check || !changed { + if options.check || (!changed && run.mode != RunMode::Compensation) { return Ok(TaskExecution::Complete { output, memory: None, @@ -4909,6 +5497,13 @@ fn repair_effect_is_unsafe( ))) } +const fn compensation_relevant_effect(effect_class: EffectClass) -> bool { + !matches!( + effect_class, + EffectClass::Pure | EffectClass::Observe | EffectClass::Model + ) +} + fn unresolved_reuse_effects( store: &SqliteStore, task: &TaskRecord, @@ -6666,7 +7261,6 @@ mod tests { network_requirements: Vec::new(), approval: ApprovalRequirement::Never, observability: Value::Null, - compensation: None, }, malformed, delay: Duration::ZERO, @@ -7410,6 +8004,7 @@ spec: &serde_json::json!({"value": "old"}), RunMode::Execute, None, + None, base, FixedClock.now(), "trace-source", @@ -7671,6 +8266,7 @@ spec: &serde_json::json!({}), RunMode::Execute, None, + None, directory.path(), FixedClock.now(), "trace", @@ -11629,6 +12225,709 @@ spec: )); } + #[tokio::test] + async fn compensation_runs_in_reverse_order_reconciles_and_replays() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: compensation } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:write + with: { path: first.txt, content: created-first } + compensate: + uses: action:write + with: { path: first.txt, content: compensated-first } + - id: second + uses: action:write + needs: [first] + with: { path: second.txt, content: created-second } + compensate: + uses: action:write + with: { path: second.txt, content: compensated-second } + - id: fail + uses: action:assert + needs: [second] + with: { that: false, message: expected } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected failed source, got {other:?}"), + }; + let compensation_plan = runtime + .plan_compensation(&source_run_id, &[]) + .expect("compensation plan"); + assert!(compensation_plan.executable); + assert!(compensation_plan.complete); + assert_eq!( + compensation_plan + .tasks + .iter() + .map(|task| task.source_task_id.as_str()) + .collect::>(), + ["second", "first"] + ); + + let outcome = runtime + .compensate( + compensation_plan, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("compensation"); + assert_eq!(outcome.state, Some(RunState::Succeeded)); + assert_eq!(outcome.compensated_tasks, ["second", "first"]); + assert_eq!( + std::fs::read_to_string(directory.path().join("first.txt")).expect("first"), + "compensated-first" + ); + assert_eq!( + std::fs::read_to_string(directory.path().join("second.txt")).expect("second"), + "compensated-second" + ); + let compensation_run_id = outcome.run_id.expect("compensation run"); + let compensation_run = store + .load_run(&compensation_run_id) + .expect("compensation record"); + assert_eq!(compensation_run.mode, RunMode::Compensation); + assert_eq!( + compensation_run.source_run_id.as_deref(), + Some(source_run_id.as_str()) + ); + assert_eq!( + store + .audit_events(&compensation_run_id) + .expect("compensation audit") + .iter() + .filter(|event| event.event_type == "effect.requested") + .filter_map(|event| event.task_id.as_deref()) + .collect::>(), + ["compensate--second", "compensate--first"] + ); + for effect in store.list_effects(&source_run_id).expect("source effects") { + if compensation_relevant_effect(effect.request.effect_class) { + let reconciliation = store + .latest_effect_reconciliation(&effect.request.id) + .expect("reconciliation") + .expect("compensated"); + assert_eq!(reconciliation.status, ReconciliationStatus::Compensated); + assert!(reconciliation.compensation_effect_id.is_some()); + } + } + + let repeated = runtime + .plan_compensation(&source_run_id, &[]) + .expect("repeat plan"); + assert!(!repeated.executable); + assert!(repeated.complete); + assert_eq!(repeated.already_compensated_effects.len(), 2); + + let blocked_retry = runtime + .plan_retry(&source_run_id, &workflow, &plan, &[], true, false) + .expect("failed-only retry plan"); + assert!(!blocked_retry.compatible); + let safe_retry = runtime + .plan_retry( + &source_run_id, + &workflow, + &plan, + &["first".to_owned()], + false, + true, + ) + .expect("explicit restart retry"); + assert!(safe_retry.compatible); + assert_eq!(safe_retry.rerun_tasks, ["first", "second", "fail"]); + + let mut repaired_workflow = workflow.clone(); + repaired_workflow.spec.tasks[2] + .input + .insert("that".to_owned(), Value::Bool(true)); + let repaired_plan = + compile(&repaired_workflow, "repaired.yaml").expect("repaired workflow compiles"); + let safe_repair = runtime + .plan_repair( + &source_run_id, + &repaired_workflow, + &repaired_plan, + &["first".to_owned()], + true, + ) + .expect("explicit restart repair"); + assert!(safe_repair.compatible); + assert_eq!(safe_repair.rerun_tasks, ["first", "second", "fail"]); + + let replay = runtime + .replay(&compensation_run_id) + .await + .expect("offline compensation replay"); + assert_eq!(replay.state, RunState::Succeeded); + } + + #[tokio::test] + async fn compensation_continues_after_failure_and_retries_only_remaining_effects() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: partial-compensation } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: first + uses: action:write + with: { path: first.txt, content: created-first } + compensate: + uses: action:write + with: { path: first.txt, content: compensated-first } + - id: second + uses: action:write + needs: [first] + with: { path: second.txt, content: created-second } + compensate: + uses: action:write + with: { path: blocked/result.txt, content: compensated-second } + retry: { maxAttempts: 2 } + - id: fail + uses: action:assert + needs: [second] + with: { that: false } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected failed source, got {other:?}"), + }; + std::fs::write(directory.path().join("blocked"), "not-a-directory").expect("blocking file"); + let outcome = runtime + .compensate( + runtime + .plan_compensation(&source_run_id, &[]) + .expect("plan"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("partial compensation"); + assert_eq!(outcome.state, Some(RunState::Failed)); + assert_eq!(outcome.compensated_tasks, ["first"]); + assert_eq!(outcome.failed_tasks, ["second"]); + let compensation_run = outcome.run_id.expect("compensation run"); + assert_eq!( + store + .list_tasks(&compensation_run) + .expect("tasks") + .into_iter() + .find(|task| task.task_id == "compensate--second") + .expect("failed compensation") + .attempt, + 2 + ); + + std::fs::remove_file(directory.path().join("blocked")).expect("remove blocker"); + let retry_plan = runtime + .plan_compensation(&source_run_id, &[]) + .expect("retry plan"); + assert_eq!( + retry_plan + .tasks + .iter() + .map(|task| task.source_task_id.as_str()) + .collect::>(), + ["second"] + ); + let retry = runtime + .compensate(retry_plan, RunOptions::default(), &CancellationToken::new()) + .await + .expect("compensation retry"); + assert_eq!(retry.state, Some(RunState::Succeeded)); + assert_eq!(retry.compensated_tasks, ["second"]); + assert_eq!( + runtime + .plan_compensation(&source_run_id, &[]) + .expect("complete") + .already_compensated_effects + .len(), + 2 + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn uncertain_compensation_blocks_repeat_until_reconciled() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: uncertain-compensation } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + processAllowlist: [sh] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + uncertain: + kind: builtin.shell.exec + command: /bin/sh + args: [-c, "sleep 2"] + timeoutSeconds: 1 + tasks: + - id: create + uses: action:write + with: { path: resource.txt, content: created } + compensate: + uses: action:uncertain + - id: fail + uses: action:assert + needs: [create] + with: { that: false } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected failed source, got {other:?}"), + }; + let outcome = runtime + .compensate( + runtime + .plan_compensation(&source_run_id, &[]) + .expect("initial plan"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("uncertain compensation outcome"); + assert_eq!(outcome.state, Some(RunState::Failed)); + let compensation_run_id = outcome.run_id.expect("compensation run"); + let compensation_effect = store + .list_effects(&compensation_run_id) + .expect("compensation effects") + .pop() + .expect("uncertain compensation effect"); + assert_eq!(compensation_effect.status, EffectStatus::Uncertain); + + let blocked = runtime + .plan_compensation(&source_run_id, &[]) + .expect("blocked repeat plan"); + assert!(!blocked.executable); + assert!(blocked.blocked.iter().any(|block| { + block.task_id == "create" + && block.rule == "unreconciled_compensation_effect" + && block.effect_id.as_deref() == Some(compensation_effect.request.id.as_str()) + })); + + store + .reconcile_effect( + &EffectReconciliationRequest { + reconciliation_id: "compensation-applied".to_owned(), + effect_id: compensation_effect.request.id.clone(), + status: ReconciliationStatus::Applied, + actor: "operator".to_owned(), + reason: "external evidence confirms inverse application".to_owned(), + evidence: serde_json::json!({"ticket": "INC-2"}), + result: Some(serde_json::json!({"status": "applied"})), + result_schema: None, + authorization: serde_json::json!({"kind": "test-operator"}), + compensation_effect_id: None, + trace_id: "trace-compensation-reconciliation".to_owned(), + }, + FixedClock.now(), + ) + .expect("reconcile compensation applied"); + let complete = runtime + .plan_compensation(&source_run_id, &[]) + .expect("complete after reconciliation"); + assert!(!complete.executable); + assert!(complete.complete); + assert_eq!(complete.already_compensated_effects.len(), 1); + let source_effect = store + .list_effects(&source_run_id) + .expect("source effects") + .into_iter() + .find(|effect| compensation_relevant_effect(effect.request.effect_class)) + .expect("source effect"); + assert_eq!( + store + .latest_effect_reconciliation(&source_effect.request.id) + .expect("source reconciliation") + .expect("compensated source") + .compensation_effect_id + .as_deref(), + Some(compensation_effect.request.id.as_str()) + ); + } + + #[tokio::test] + async fn cancelled_compensation_dispatches_no_effect_and_remains_retryable() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: cancelled-compensation } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: create + uses: action:write + with: { path: resource.txt, content: created } + compensate: + uses: action:write + with: { path: resource.txt, content: compensated } + - id: fail + uses: action:assert + needs: [create] + with: { that: false } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected failed source, got {other:?}"), + }; + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let outcome = runtime + .compensate( + runtime + .plan_compensation(&source_run_id, &[]) + .expect("initial plan"), + RunOptions::default(), + &cancellation, + ) + .await + .expect("cancelled outcome"); + assert_eq!(outcome.state, Some(RunState::Cancelled)); + assert!( + store + .list_effects(outcome.run_id.as_deref().expect("compensation run")) + .expect("compensation effects") + .is_empty() + ); + let retry = runtime + .plan_compensation(&source_run_id, &[]) + .expect("retryable plan"); + assert!(retry.executable); + assert!(retry.blocked.is_empty()); + } + + #[tokio::test] + async fn automatic_compensation_is_opt_in_and_uses_a_linked_run() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: automatic-compensation } +spec: + compensation: { onFailure: automatic } + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: create + uses: action:write + with: { path: resource.txt, content: created } + compensate: + uses: action:write + with: { path: resource.txt, content: compensated } + - id: fail + uses: action:assert + needs: [create] + with: { that: false } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected failed source, got {other:?}"), + }; + assert_eq!( + std::fs::read_to_string(directory.path().join("resource.txt")).expect("resource"), + "compensated" + ); + let compensation_runs = store + .compensation_runs(&source_run_id) + .expect("compensation lineage"); + assert_eq!(compensation_runs.len(), 1); + assert_eq!(compensation_runs[0].1, RunState::Succeeded); + } + + #[tokio::test] + async fn compensation_approval_is_durable_and_resume_finalizes_source_effects() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: compensation-approval } +spec: + compensation: { approval: always } + policy: + workspaceRoot: . + writableRoots: [.] + approval: never + nonInteractive: pause + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: create + uses: action:write + with: { path: resource.txt, content: created } + compensate: + uses: action:write + with: { path: resource.txt, content: compensated } + - id: fail + uses: action:assert + needs: [create] + with: { that: false } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected failed source, got {other:?}"), + }; + let paused = runtime + .compensate( + runtime + .plan_compensation(&source_run_id, &[]) + .expect("plan"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("paused compensation"); + assert_eq!(paused.state, Some(RunState::Paused)); + let compensation_run_id = paused.run_id.expect("compensation run"); + let approval = store + .pending_approvals(&compensation_run_id) + .expect("approvals") + .pop() + .expect("approval"); + store + .resolve_approval( + &approval.approval_id, + ApprovalResolution::Approved, + "operator", + "approved declared compensation", + FixedClock.now(), + ) + .expect("approve"); + let resumed = runtime + .resume( + &compensation_run_id, + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("resume compensation"); + assert_eq!(resumed.state, RunState::Succeeded); + let source_effect = store + .list_effects(&source_run_id) + .expect("source effects") + .pop() + .expect("source effect"); + assert_eq!( + store + .latest_effect_reconciliation(&source_effect.request.id) + .expect("reconciliation") + .expect("compensated") + .status, + ReconciliationStatus::Compensated + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn compensation_waits_for_uncertain_source_reconciliation() { + let directory = tempdir().expect("tempdir"); + let store = SqliteStore::open_memory().expect("store"); + let runtime = runtime(store.clone(), directory.path()); + let (workflow, plan) = compile_fixture( + r#" +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: reconciled-compensation } +spec: + policy: + workspaceRoot: . + writableRoots: [.] + processAllowlist: [sh] + approval: never + actions: + uncertain: + kind: builtin.shell.exec + command: /bin/sh + args: [-c, "sleep 2"] + timeoutSeconds: 1 + write: { kind: builtin.write } + tasks: + - id: uncertain + uses: action:uncertain + compensate: + uses: action:write + with: { path: reconciled.txt, content: compensated } +"#, + ); + let source_run_id = match runtime + .start( + &workflow, + &plan, + serde_json::json!({}), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + { + Err(RuntimeError::RunFailed { run_id, .. }) => run_id, + other => panic!("expected uncertain failure, got {other:?}"), + }; + let blocked = runtime + .plan_compensation(&source_run_id, &[]) + .expect("blocked plan"); + assert!(!blocked.executable); + assert!(blocked.blocked.iter().any(|block| { + block.task_id == "uncertain" && block.rule == "unreconciled_source_effect" + })); + let source_effect = store + .list_effects(&source_run_id) + .expect("effects") + .pop() + .expect("uncertain effect"); + store + .reconcile_effect( + &EffectReconciliationRequest { + reconciliation_id: "source-applied".to_owned(), + effect_id: source_effect.request.id, + status: ReconciliationStatus::Applied, + actor: "operator".to_owned(), + reason: "external evidence confirms application".to_owned(), + evidence: serde_json::json!({"ticket": "INC-1"}), + result: Some(serde_json::json!({"status": "applied"})), + result_schema: None, + authorization: serde_json::json!({"kind": "test-operator"}), + compensation_effect_id: None, + trace_id: "trace-source-reconciliation".to_owned(), + }, + FixedClock.now(), + ) + .expect("reconcile source applied"); + let compensation = runtime + .compensate( + runtime + .plan_compensation(&source_run_id, &[]) + .expect("unblocked plan"), + RunOptions::default(), + &CancellationToken::new(), + ) + .await + .expect("compensate reconciled source"); + assert_eq!(compensation.state, Some(RunState::Succeeded)); + assert_eq!( + std::fs::read_to_string(directory.path().join("reconciled.txt")) + .expect("compensated output"), + "compensated" + ); + } + #[test] fn subprocess_output_redaction_removes_every_known_secret_value() { let output = redact_text("token=top-secret; repeated=top-secret", &["top-secret"]); diff --git a/crates/agentctl-store/src/lib.rs b/crates/agentctl-store/src/lib.rs index 28fec60..623bc22 100644 --- a/crates/agentctl-store/src/lib.rs +++ b/crates/agentctl-store/src/lib.rs @@ -400,6 +400,7 @@ pub enum RunMode { Fork, Repair, Retry, + Compensation, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -1016,6 +1017,7 @@ impl SqliteStore { working_memory: &Value, mode: RunMode, parent_run_id: Option<&str>, + source_run_id: Option<&str>, base_path: &Path, now: DateTime, trace_id: &str, @@ -1023,7 +1025,7 @@ impl SqliteStore { let mut connection = self.connection.lock(); let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; transaction.execute( - "INSERT INTO runs (run_id, runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, state, mode, parent_run_id, base_path, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?15)", + "INSERT INTO runs (run_id, runtime_state_version, workflow_digest, workflow_schema_version, plan_digest, plan_format_version, workflow_json, plan_json, inputs_json, working_memory_json, state, mode, parent_run_id, source_run_id, base_path, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?16)", params![ run_id, RUNTIME_STATE_VERSION, @@ -1042,6 +1044,7 @@ impl SqliteStore { encode_enum(RunState::Running)?, encode_enum(mode)?, parent_run_id, + source_run_id, base_path.display().to_string(), now.to_rfc3339(), ], @@ -1061,7 +1064,11 @@ impl SqliteStore { "run.created", None, trace_id, - &serde_json::json!({"mode": mode, "planDigest": plan.plan_digest}), + &serde_json::json!({ + "mode": mode, + "planDigest": plan.plan_digest, + "sourceRunId": source_run_id, + }), now, &self.protection, )?; @@ -1516,6 +1523,28 @@ impl SqliteStore { }) } + pub fn compensation_runs( + &self, + source_run_id: &str, + ) -> Result, StoreError> { + let connection = self.connection.lock(); + let mut statement = connection.prepare( + "SELECT run_id, state FROM runs + WHERE source_run_id = ?1 AND mode = ?2 + ORDER BY created_at, run_id", + )?; + statement + .query_map( + params![source_run_id, encode_enum(RunMode::Compensation)?], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + )? + .map(|row| { + let (run_id, state) = row?; + Ok((run_id, decode_enum(&state, "run.state")?)) + }) + .collect() + } + pub fn record_replay_effects_reused( &self, replay_run_id: &str, @@ -2623,17 +2652,32 @@ impl SqliteStore { "an effect cannot compensate itself".to_owned(), )); } - let compensation: (String, String, bool) = transaction + let compensation: (String, String, bool, Option, String) = transaction .query_row( - "SELECT run_id, status, confirmed FROM effects WHERE effect_id = ?1", + "SELECT e.run_id, e.status, e.confirmed, r.source_run_id, r.mode + FROM effects e + JOIN runs r ON r.run_id = e.run_id + WHERE e.effect_id = ?1", [compensation_effect_id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, ) .optional()? .ok_or_else(|| StoreError::EffectNotFound(compensation_effect_id.clone()))?; - if compensation.0 != source.0 { + let compensation_mode: RunMode = decode_enum(&compensation.4, "compensation_run.mode")?; + let linked_compensation_run = compensation_mode == RunMode::Compensation + && compensation.3.as_deref() == Some(source.0.as_str()); + if compensation.0 != source.0 && !linked_compensation_run { return Err(StoreError::Incompatible( - "compensation effect must belong to the same run".to_owned(), + "compensation effect must belong to the same run or a source-linked compensation run" + .to_owned(), )); } let compensation_status: EffectStatus = @@ -4624,6 +4668,7 @@ spec: &serde_json::json!({}), RunMode::Execute, None, + None, Path::new("."), Utc::now(), "trace", @@ -4772,6 +4817,7 @@ spec: &serde_json::json!({"working": marker}), RunMode::Execute, None, + None, directory.path(), Utc::now(), "trace-encryption", diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f8dca98..03cbf95 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -42,10 +42,11 @@ iteration attempts, effects, guard decisions, retry, repair, and replay use the ordinary durable task model. Reusable sub-workflows compile into a typed input boundary, namespaced ordinary tasks, and a typed output aggregate. Their policy and providers come from the invoking workflow, while deterministic memory keys -are invocation-prefixed. Handlers, compensation execution, and event triggers -still require their own explicit state and recovery contracts. The DSL carries -optional compensation metadata on a tool contract, but the runtime does not -execute compensation. +are invocation-prefixed. Explicit compensation plans eligible applied effects +in reverse graph order and executes ordinary actions in a separate +source-linked run. Confirmed inverse effects append immutable reconciliation +records to the source. Handlers and event triggers remain outside the runtime +surface. Clock and identifier generation are injected. Provider responses, tools, and external actions are injected interfaces. Cryptographic digests canonicalize identity; output maps use stable ordering where the public contract requires it. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 0e78e6a..7e2b86a 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -18,8 +18,12 @@ Schema 5 adds selective-repair metadata without changing resume, replay, retry, Unversioned YAML is compatibility-only and warns. The TypeScript package exposes no `bin` or `main` and is archived. Placeholder memory adapters, provider environment-name-only “support,” YAML output, legacy profiles, automatic endpoint overrides, old prompt-cache fields, and optimistic replay semantics are removed from production. +Tool-level `compensation` metadata was never executable and is rejected. Declare +an effectful inverse action on each source task with `compensate`; see +[Compensation](guides/COMPENSATION.md). + Legacy workflows depending on packs, broad built-in tool profiles, remote MCP/A2A shape, MongoDB memory, provider-specific endpoint fields, or embedded credentials require manual conversion. The translator intentionally refuses to guess security-sensitive intent. ## Separate product decisions -Teams/handoffs, compensation execution, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. Bounded loops and namespaced sub-workflows are additive; unbounded or model-controlled graph growth is intentionally unsupported. +Teams/handoffs, a public pack registry/resolver, vector memory, automatic MCP reconnection, general A2A resubmission, and streamed model output are not compatibility promises for v1alpha1. Bounded loops, namespaced sub-workflows, and explicit source-linked compensation are additive; unbounded or model-controlled graph growth is intentionally unsupported. diff --git a/docs/DSL.md b/docs/DSL.md index c419bc5..36adfce 100644 --- a/docs/DSL.md +++ b/docs/DSL.md @@ -2,7 +2,7 @@ The current document version is `agentctl.dev/v1alpha1`, with `kind: Workflow`. The generated, authoritative JSON Schema is [`schemas/workflow.schema.json`](../schemas/workflow.schema.json). YAML documents are limited to 1 MiB and reject unknown fields. -`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; reusable sub-workflows; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, `workflow:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach`, `matrix`, or `loop` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, and failure behavior. +`metadata` contains the name, description, and labels. `spec` contains typed inputs/outputs; providers; bounded agents; actions; tool contracts; reusable sub-workflows; compensation policy; ordered tasks; policy; memory; MCP servers; A2A peers; packs; runtime; and output settings. A task `uses` `action:`, `agent:`, `workflow:`, or the pure `router` construct. Tasks declare `needs`, optional bounded `foreach`, `matrix`, or `loop` expansion, optional working-memory `memoryWrites`, an optional `when`, local `vars`, typed `with` input, optional `outputSchema`, retry, timeout, failure behavior, and an optional effectful `compensate` action. Templates use only `${{ inputs.path }}`, `${{ vars.path }}`, `${{ memory.path }}`, and `${{ tasks.task-id.output.path }}`. Conditions additionally allow `not` and equality against a JSON literal or string. Exact templates preserve their JSON type; interpolation into text accepts only scalars. Missing and explicit `null` are different. There is no code execution, function call, indexing, arithmetic, or implicit task dependency. @@ -43,8 +43,14 @@ a typed input boundary, `INVOCATION--LOCAL_TASK` children, and a typed output aggregate. Pack manifests export the same contract under `workflows`. See [Reusable sub-workflows](guides/SUB_WORKFLOWS.md). +Compensable tasks declare one named effectful action under `compensate`. +Compensation is manual unless `spec.compensation.onFailure` is `automatic`. +Planning uses confirmed source effects, runs inverse actions in reverse graph +order, and appends linked `compensated` reconciliation records. See +[Compensate applied effects](guides/COMPENSATION.md). + `builtin.shell.exec` captures stdout and stderr concurrently. Its optional `stdoutLimitBytes`, `stderrLimitBytes`, and `combinedOutputLimitBytes` fields default to 1 MiB, 1 MiB, and 2 MiB respectively. Each configured value must be between 1 byte and 16 MiB. `timeoutSeconds` must be between 1 and 86,400. Exceeding an output bound terminates and reaps the process and records a structured failed effect; timeout or cancellation remains an uncertain effect because external changes may already have occurred. These fields are validated identically for workflow and pack actions. The parser translates a limited unversioned `playbook:` document and emits a migration warning. Use `agentctl migrate old.yaml --write new.yaml`. Legacy pack-backed, MCP, A2A, provider-specific, and broad module configurations need manual migration; see [Migrating from TypeScript](MIGRATING_FROM_TYPESCRIPT.md). -Not implemented in v1alpha1: `finally`, handlers, event triggers, or compensation execution. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. +Not implemented in v1alpha1: `finally`, handlers, or event triggers. Parallelism is expressed by independent graph tasks rather than a separate parallel-group construct. diff --git a/docs/DURABLE_EXECUTION.md b/docs/DURABLE_EXECUTION.md index 762190f..74401e7 100644 --- a/docs/DURABLE_EXECUTION.md +++ b/docs/DURABLE_EXECUTION.md @@ -8,6 +8,7 @@ SQLite is the local history and correctness boundary. Run, task, effect, approva - Recorded replay creates a replay record from terminal stored outputs and calls no provider, tool, network, process, or filesystem executor. - Terminal retry creates a new source-linked run for an identical workflow, materializes compatible successful boundaries, and executes failed or explicitly selected roots plus their descendants with fresh attempts. - Selective repair creates a new source-linked run, materializes compatible successful task outputs and committed state deltas, then executes selected roots and descendants with fresh effects from a target workflow. +- Compensation creates a source-linked sequential run for explicitly declared inverse actions. Confirmed inverse effects append `compensated` reconciliations to immutable source effects; partial failures remain independently retryable. - Fork creates a new run linked to the old run and intentionally permits fresh effects. - A task's `retry` policy creates another attempt inside the same run only within its explicit bound. An unsafe unresolved effect is not retried. @@ -53,4 +54,12 @@ tasks surround namespaced child tasks, so child attempts, effects, artifacts, approvals, retry/repair lineage, cancellation, and replay stay in the ordinary run graph. +Compensation planning is effect-free. Only confirmed successful mutations or +effects reconciled as applied are eligible. Started and uncertain effects +require operator reconciliation. The generated compensation run uses reverse +compiled order, ordinary effect identities, policy, approvals, retries, +checkpoints, audit, and traces. A repeated plan excludes source effects already +reconciled as compensated. This is best-effort inverse execution, not +transactional rollback. + The artifact root is `artifacts/` beside the database. `agentctl artifacts` lists references and blobs, verifies hashes, exports bytes atomically, and performs reachability-based collection. GC excludes referenced blobs and active ingestion leases, recovers interrupted quarantine operations on startup, and cleans stale untracked blobs and partial temporary files. diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index e1124ac..f803ca6 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -23,7 +23,6 @@ No known P0/P1 implementation defect remains for the stated local, scheduled, an These are useful extensions but are not required by the product thesis. They need new deterministic state and compatibility contracts before implementation: -- compensation execution; - structured agent teams and handoffs; - model token streaming into CLI/workflow state; - opt-in MCP reconnection and A2A resubmission with explicit remote reconciliation; @@ -56,6 +55,10 @@ These are useful extensions but are not required by the product thesis. They nee - Sub-workflows are compile-time namespaced graphs with semantic versions and typed input/output boundaries. Definitions inherit the caller's policy and providers and cannot request independent authority. +- Compensation is explicit best-effort inverse execution. It runs as a + source-linked sequential workflow, skips effects already reconciled as + compensated, and never claims transactional rollback or exactly-once + external mutation. - SQLite is local durable state, not a secret vault or distributed lease service. Persist `/state` across container invocations and back it up according to the workflow's recovery needs. - State encryption is explicit and selected-field only. Before it is enabled, the database is plaintext. It does not encrypt artifact bytes or operational metadata, and it cannot retroactively protect old backups or snapshots. Preserve the current referenced key with encrypted backups. - Filesystem/process/network allowlists are not an OS sandbox. Run untrusted workflows in a restricted container/VM with least-privilege credentials and egress. diff --git a/docs/TOOLS.md b/docs/TOOLS.md index cbb3eb8..0c1e3b2 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -1,6 +1,6 @@ # Tools and effects -A tool contract has a stable ID, description, input and output JSON Schema, capability, risk, effect class, idempotency, retry-safety flag, timeout, secret and network requirements, approval mode, and optional compensation metadata. Inputs are validated before an executor is called; outputs are validated before entering messages or task state. Executor errors remain errors. +A tool contract has a stable ID, description, input and output JSON Schema, capability, risk, effect class, idempotency, retry-safety flag, timeout, secret and network requirements, and approval mode. Inputs are validated before an executor is called; outputs are validated before entering messages or task state. Executor errors remain errors. Effect classes are `pure`, `internal_state`, `observe`, `workspace_mutate`, `external_mutate`, `process_execution`, `network`, `model`, and `remote_agent`. Idempotency is `pure`, `idempotent`, `keyed`, `at_most_once`, or `unknown`. These values drive durable recovery and policy; model-provided MCP annotations never override them. diff --git a/docs/adr/0013-compensation-as-source-linked-runs.md b/docs/adr/0013-compensation-as-source-linked-runs.md new file mode 100644 index 0000000..1d704d8 --- /dev/null +++ b/docs/adr/0013-compensation-as-source-linked-runs.md @@ -0,0 +1,36 @@ +# ADR 0013: Compensation as source-linked runs + +- Status: accepted +- Date: 2026-07-24 + +## Context + +Applied external mutations sometimes need an explicit inverse operation. +Mutating a terminal source run would weaken run immutability, while pretending +that inverse operations form a transaction would overstate external-system +guarantees. + +## Decision + +Each compensable task declares a named effectful action and bounded execution +settings. The runtime plans from durable source effects, orders eligible tasks +in reverse compiled graph order, and executes them as a separate sequential +run with `mode: compensation` and `sourceRunId`. + +Compensation uses ordinary tasks, policy, approvals, effect identities, +uncertainty handling, retries, cancellation, checkpoints, audit, and traces. +Successful compensation effects append `compensated` reconciliation records +to immutable source effects. A repeated plan excludes those effects. + +Manual execution is the default. Automatic execution occurs only when +`compensation.onFailure` is `automatic`. + +## Consequences + +- The source run and source effects remain immutable. +- Partial compensation is durable and retryable without repeating completed + inverse effects. +- An uncertain source or compensation effect requires reconciliation. +- Retry and repair cannot reuse a task whose applied effect was compensated. +- Compensation remains an honest best-effort operation, not rollback, + exactly-once delivery, or a distributed transaction. diff --git a/docs/execution/COMPATIBILITY.md b/docs/execution/COMPATIBILITY.md index 30fc97f..1842926 100644 --- a/docs/execution/COMPATIBILITY.md +++ b/docs/execution/COMPATIBILITY.md @@ -26,7 +26,13 @@ The TypeScript oracle is commit `be9d0ae`; the detailed public policy is [docs/C ## Removed - Direct API-key flags, YAML machine output, legacy profiles, placeholder provider/memory support, optimistic replay, implicit full environment inheritance, and obsolete provider/cache fields. +- Non-executable tool-level `compensation` metadata. Declare the inverse action + on an effectful task with `compensate`. -## Manual migration/deferred +## Manual migration -Legacy pack-backed workflows, custom TypeScript executors, MongoDB/vector memory, and old MCP/A2A shapes require manual conversion. Parallel/dynamic workflows, sub-workflows, teams/handoffs, compensation execution, registry resolution, and automatic remote resubmission are deferred product decisions. +Legacy custom TypeScript executors, MongoDB memory, and old MCP/A2A shapes +require manual conversion. Bounded parallel and dynamic tasks, sub-workflows, +and source-linked compensation are additive. A public registry is an explicit +non-goal. Structured handoffs and safe remote continuation remain tracked in +the limitation burn-down until their product paths are verified. diff --git a/docs/execution/COMPLETENESS_VERIFICATION.md b/docs/execution/COMPLETENESS_VERIFICATION.md index 1a23ef8..492b52a 100644 --- a/docs/execution/COMPLETENESS_VERIFICATION.md +++ b/docs/execution/COMPLETENESS_VERIFICATION.md @@ -79,7 +79,8 @@ cargo xtask acceptance-container | Conditions/routers | compiler typed-case/guard failures and runtime durable condition, route, retry, changed-input repair, and skipped replay tests passed | packaged CLI scenario 35 passed | deterministic verified; live pending | | Bounded loops | compiler bounds/identity tests and runtime zero/one/max, exhaustion, cancellation, uncertain effect, retry, repair, and replay tests passed | packaged CLI scenario 36 passed | deterministic verified; live pending | | Sub-workflows | compiler namespacing/version/cycle/state-isolation tests and runtime typed boundary, retry, repair, and replay tests passed | packaged CLI scenario 37 and integrity-pinned pack example passed | deterministic verified; live pending | -| Compensation/handoffs/streaming | pending | pending | open | +| Compensation | compiler declarations plus runtime reverse order, approval, partial failure, source and inverse uncertainty, cancellation, retry, reconciliation, automatic trigger, repair/retry invalidation, and replay tests passed | packaged CLI scenario 38 and the full local release gate passed | verified | +| Handoffs/streaming | pending | pending | open | | MCP/A2A resilience | pending | pending | open | | Packs/trust/extensions | pending | pending | open | | Semantic memory | pending | pending | open | diff --git a/docs/execution/DECISIONS.md b/docs/execution/DECISIONS.md index 450f0b8..80703b0 100644 --- a/docs/execution/DECISIONS.md +++ b/docs/execution/DECISIONS.md @@ -14,5 +14,6 @@ | [0010](../adr/0010-typed-routing-and-durable-decisions.md) | Typed routing and durable decisions | accepted | Pure enumerated routers and hashed condition contexts make branching inspectable and replayable. | | [0011](../adr/0011-bounded-loops-as-static-graphs.md) | Bounded loops as static graphs | accepted | Fixed iteration chains reuse ordinary durable task and recovery semantics. | | [0012](../adr/0012-subworkflows-as-namespaced-graphs.md) | Sub-workflows as namespaced graphs | accepted | Typed boundaries and flattened children avoid a hidden nested runtime. | +| [0013](../adr/0013-compensation-as-source-linked-runs.md) | Compensation as source-linked runs | accepted | Immutable source effects gain explicit best-effort inverse lineage without pretending to be transactions. | These decisions resolve the researched patterns in [LANDSCAPE.md](../research/LANDSCAPE.md). No unsafe code or distributed control plane ADR is required because neither exists. diff --git a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md index 16c07d0..141bd6e 100644 --- a/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md +++ b/docs/execution/EXAMPLE_VERIFICATION_MATRIX.md @@ -30,6 +30,7 @@ This inventory is enforced by `cargo xtask examples-verify`. The default command | `examples/v1/approval.yaml` | Approval-paused mutation | deterministic | paused | 0 | 0 | Acceptance equivalent | N/A | N/A | N/A | No write before approval | Canonical | passed | | `examples/v1/capability-failure.yaml` | Negative capability contract | deterministic | validation failure | 2 | 2 | Expected failure | N/A | N/A | N/A | N/A | JSON diagnostics | passed | | `examples/v1/check-diff.yaml` | Non-mutating check and diff | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | No mutation | Canonical | passed | +| `examples/v1/compensation.yaml` | Explicit source-linked best-effort compensation | deterministic | expected source failure then compensation | 0 | 0 | Acceptance scenario 38 | N/A | N/A | Pending composite OCI | Source effect reconciliation | Reverse order and compensated file | passed | | `examples/v1/condition.yaml` | Conditional scheduling | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | | `examples/v1/crash-resume.yaml` | Durable interruption fixture | fake | resumable | 0 | 0 | Acceptance equivalent | Canonical | N/A | N/A | N/A | Durable state | passed | | `examples/v1/dataflow.yaml` | Typed task dataflow | deterministic | success | 0 | 0 | Canonical | N/A | N/A | N/A | N/A | Canonical | passed | diff --git a/docs/execution/LIMITATION_BURNDOWN.md b/docs/execution/LIMITATION_BURNDOWN.md index e84d825..5181f05 100644 --- a/docs/execution/LIMITATION_BURNDOWN.md +++ b/docs/execution/LIMITATION_BURNDOWN.md @@ -48,7 +48,7 @@ complete, every entry must have exactly one final disposition: | COND-001 | Conditions and routers | in progress | implemented | | LOOP-001 | Bounded loops | in progress | implemented | | SUB-001 | Sub-workflows | in progress | implemented | -| COMP-001 | Compensation | open | implemented | +| COMP-001 | Compensation | in progress | implemented | | TEAM-001 | Structured teams and handoffs | open | redesigned | | STR-001 | Streaming | open | implemented | | MCP-001 | MCP resilience | open | implemented | @@ -350,23 +350,35 @@ complete, every entry must have exactly one final disposition: ### COMP-001: Explicit compensation -- Current behavior: tool contracts carry compensation metadata but runtime does - not execute it. +- Current behavior: compensable tasks declare a named effectful inverse action. + `agentctl compensate` plans and executes eligible source effects through a + separate source-linked run. - User impact: operators cannot durably coordinate best-effort reversal. - Security or durability impact: documentation-shaped metadata can be mistaken for transactional rollback. -- Product decision: compensation is an explicit new run phase in reverse - dependency order, never a transactional rollback claim. +- Product decision: compensation is an explicit sequential run in reverse + compiled graph order, never a transactional rollback claim. - Required implementation: declaration validation, manual trigger, opt-in automatic trigger, approval, idempotency, partial failure, linkage to effects and reconciliation, audit, trace, retry/repair behavior. -- Migration impact: effect links, run phase, and checkpoints. -- Tests: order, approval, idempotency, partial failure, contradictory - reconciliation, cancellation, and replay. +- Migration impact: the additive `compensation` run mode reuses existing source + lineage, task, effect, approval, checkpoint, audit, trace, and reconciliation + storage. No database schema migration is required. +- Tests: order, approval, idempotency, partial failure, source and inverse + uncertainty reconciliation, terminal inverse-run blocking, cancellation, + retry/repair invalidation, and replay. - Examples: operational workflow compensation. - Live evidence: deterministic tool compensation only. - Documentation: guarantees and non-guarantees. -- Final disposition: pending implementation evidence. +- Final disposition: implemented. Source effects remain immutable; eligible + confirmed mutations execute declared action-based compensation in reverse + order. Successful inverse effects append linked `compensated` + reconciliations. Manual and explicitly automatic triggers, approval, + uncertainty blocking, bounded retries, partial continuation, repeat planning, + selected tasks, repair/retry invalidation, and effect-free replay use the + ordinary durable runtime. Focused compiler/runtime tests, all 38 packaged CLI + scenarios, the 12-stage verification gate, examples, docs, packaging, and + secret scanning pass. ### TEAM-001: Structured teams and handoffs diff --git a/docs/generated/CLI.md b/docs/generated/CLI.md index cc11b03..b89baa6 100644 --- a/docs/generated/CLI.md +++ b/docs/generated/CLI.md @@ -18,6 +18,7 @@ Commands: fork Create a new run from a prior workflow with fresh effects repair Create a new run that reuses compatible upstream results and executes a repaired suffix retry Retry failed or selected boundaries of an identical terminal workflow + compensate Execute explicitly declared best-effort compensation for a terminal run runs Analyze or upgrade retained legacy run records for selective reuse cancel Durably request cancellation inspect Inspect durable run, task, and audit state diff --git a/docs/guides/COMPENSATION.md b/docs/guides/COMPENSATION.md new file mode 100644 index 0000000..2d38626 --- /dev/null +++ b/docs/guides/COMPENSATION.md @@ -0,0 +1,107 @@ +# Compensate applied effects + +Compensation is an explicit best-effort workflow operation. It does not provide +transactional rollback or exactly-once mutation of an external system. + +Declare one named effectful action on each task that can be compensated: + +```yaml +spec: + compensation: + onFailure: manual + approval: policy + actions: + provision: + kind: builtin.write + deprovision: + kind: builtin.write + tasks: + - id: provision + uses: action:provision + with: + path: artifacts/resource.txt + content: provisioned + compensate: + uses: action:deprovision + with: + path: "${{ tasks.provision.output.path }}" + content: compensated + retry: + maxAttempts: 2 + backoffMs: 100 +``` + +`compensate.uses` must name an effectful action. Its input can use the original +run's durable `inputs`, task outputs, task variables, and working memory. The +compiler includes the declaration in task identity, validates every reference, +and copies it to bounded matrix, foreach, loop, and sub-workflow children. + +## Plan and execute + +```console +agentctl compensate SOURCE_RUN_ID --db .agentctl/runtime.db --plan +agentctl compensate SOURCE_RUN_ID --db .agentctl/runtime.db +agentctl compensate SOURCE_RUN_ID --task provision --db .agentctl/runtime.db +``` + +The source must be terminal. Planning examines its immutable effects: + +- confirmed successful mutations and effects reconciled as `applied` are + eligible; +- `not_applied` effects require no compensation; +- already `compensated` effects are never repeated; +- started or uncertain mutations block their task until an operator reconciles + external reality. + +Eligible tasks are emitted in reverse compiled graph order. The compensation +run is source-linked, sequential, and uses the original policy, actions, +protocol configuration, working-memory snapshot, and workspace boundary. +Compensation tasks continue after a sibling failure, so successful and failed +undo actions remain individually inspectable. + +Each successful compensation effect appends a `compensated` reconciliation to +the source effect. The record links both run IDs, both task IDs, and the +compensation effect ID. Source effects and the terminal source run remain +unchanged. + +## Approval and automatic execution + +Manual execution is the default. Automatic failure handling must be explicit: + +```yaml +spec: + compensation: + onFailure: automatic + approval: always +``` + +`approval: policy` preserves the workflow policy. `always` requires durable +approval for the compensation run. `never` explicitly removes an approval +gate but does not bypass filesystem, process, network, provider, or tool +authorization. + +Automatic compensation starts only after a failed terminal run. Cancellation +does not imply automatic compensation. A paused compensation run is resumed +with the ordinary approval and `resume` commands. + +## Failure and recovery + +Every compensation effect has the ordinary durable idempotency key and +at-most-once uncertainty behavior. Definitive retry-safe failures use the +declared bounded retry. An uncertain compensation effect is never sent again +until it is reconciled, including after its compensation run becomes terminal. +An `applied` reconciliation finalizes the linked source effect; `not_applied` +allows a new compensation attempt. + +Running `compensate` again plans only source effects that do not already have a +confirmed compensation. This is the retry operation for a terminal partial +compensation run. Recorded replay of the compensation run dispatches no fresh +effects. + +A compensated source task is not reusable by retry or selective repair. +Restart it explicitly, together with its required closure, because its former +external result has been intentionally undone. + +Use `inspect` on both source and compensation run IDs. The source exposes +`effectReconciliations`; the compensation run exposes ordinary tasks, effects, +approvals, audit events, traces, and its `sourceRunId`. diff --git a/docs/reference/YAML.md b/docs/reference/YAML.md index 42b1bdb..e4b5e05 100644 --- a/docs/reference/YAML.md +++ b/docs/reference/YAML.md @@ -26,6 +26,7 @@ Unknown fields fail. Documents, ordinary input files, packs, direct reads, exist | `actions` | `{}` | Named deterministic or protocol actions. | | `tools` | `{}` | Strict model-callable tool contracts. | | `subworkflows` | `{}` | Semantically versioned reusable task graphs with typed input and output boundaries. | +| `compensation` | manual, policy approval | Best-effort compensation trigger and approval behavior. | | `tasks` | required list | Ordered graph nodes. | | `policy` | safe defaults | Filesystem, process, network, provider, tool, and approval rules. | | `memory` | empty | Initial working memory and optional SQLite long-term namespace. | @@ -54,6 +55,7 @@ Each task requires `id` and `uses`. `uses` is `action:name`, `agent:name`, | `outputSchema` | action-owned object or agent structured contract | Valid JSON Schema checked at task completion and selective-repair reuse. | | `retry` | bounded default | Only definitive retry-safe failures may repeat. | | `timeoutSeconds` | action or agent default | Must be within the implementation bound. | +| `compensate` | none | Named effectful action, typed `with`, bounded retry, and timeout. Valid only on a potentially mutating task. | | failure behavior | fail | Unsupported dynamic control flow is rejected. | Ready tasks are selected in YAML declaration order up to `maxConcurrency`. @@ -62,7 +64,9 @@ runtime or model-controlled expansion. Static `foreach` and `matrix` tasks compile to inspectable child tasks and a parent aggregate. Bounded loops compile to a sequential child chain and parent aggregate. Sub-workflows compile to namespaced ordinary tasks with typed input and output boundaries. There is -no handler or separate parallel group in this version. +no handler or separate parallel group in this version. Compensation is planned +after a terminal run and executes declared inverse actions in reverse graph +order through an ordinary source-linked durable run. ## Agents @@ -130,6 +134,7 @@ Related guides: [Workflow authoring](../guides/WORKFLOW_AUTHORING.md), [Matrix and foreach](../guides/MATRIX_AND_FOREACH.md), [Conditions and routers](../guides/CONDITIONS_AND_ROUTERS.md), [Bounded loops](../guides/BOUNDED_LOOPS.md), [Reusable -sub-workflows](../guides/SUB_WORKFLOWS.md), [Secret +sub-workflows](../guides/SUB_WORKFLOWS.md), +[Compensation](../guides/COMPENSATION.md), [Secret references](../guides/SECRET_REFERENCES.md), [Policies](../policies.md), [Tools](../TOOLS.md), and [Workflow DSL](../DSL.md). diff --git a/examples/v1/README.md b/examples/v1/README.md index f88af18..18c4d02 100644 --- a/examples/v1/README.md +++ b/examples/v1/README.md @@ -14,6 +14,7 @@ The deterministic examples are exercised by `cargo xtask verify` and never requi - `matrix.yaml`: bounded static matrix expansion, stable child identities, and ordered aggregation. - `loop.yaml`: bounded sequential iteration, stable boundaries, aggregation, and fail-closed exhaustion. - `subworkflow.yaml`: typed reusable graph expansion and namespaced recovery boundaries. +- `compensation.yaml`: expected downstream failure followed by explicit source-linked compensation. - `working-memory.yaml` and `long-term-memory.yaml`: separate memory lifecycles. - `fake-provider.yaml`: deterministic model-provider path. - `mcp.yaml` and `a2a.yaml`: local protocol fixtures, backed by the protocol crate's mock-server tests. diff --git a/examples/v1/compensation.yaml b/examples/v1/compensation.yaml new file mode 100644 index 0000000..10db1bc --- /dev/null +++ b/examples/v1/compensation.yaml @@ -0,0 +1,40 @@ +apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: + name: compensation + description: Run explicit best-effort compensation after a downstream failure. +spec: + compensation: + onFailure: manual + approval: policy + policy: + workspaceRoot: . + writableRoots: + - artifacts + approval: never + actions: + write: + kind: builtin.write + assert: + kind: builtin.assert + tasks: + - id: provision + uses: action:write + with: + path: artifacts/compensation-resource.txt + content: provisioned + compensate: + uses: action:write + with: + path: artifacts/compensation-resource.txt + content: compensated + retry: + maxAttempts: 2 + backoffMs: 50 + - id: validate + uses: action:assert + needs: + - provision + with: + that: false + message: expected failure demonstrates manual compensation diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json index b5e55da..5b115f3 100644 --- a/schemas/workflow.schema.json +++ b/schemas/workflow.schema.json @@ -102,6 +102,13 @@ }, "default": {} }, + "compensation": { + "$ref": "#/$defs/CompensationPolicyDefinition", + "default": { + "onFailure": "manual", + "approval": "policy" + } + }, "tasks": { "type": "array", "items": { @@ -622,12 +629,6 @@ "approval": { "$ref": "#/$defs/ApprovalRequirement", "default": "policy" - }, - "compensation": { - "type": [ - "string", - "null" - ] } }, "additionalProperties": false, @@ -825,6 +826,16 @@ "$ref": "#/$defs/FailureBehavior", "default": "stop" }, + "compensate": { + "anyOf": [ + { + "$ref": "#/$defs/CompensationDefinition" + }, + { + "type": "null" + } + ] + }, "outputSchema": true }, "additionalProperties": false, @@ -949,6 +960,60 @@ "continue" ] }, + "CompensationDefinition": { + "type": "object", + "properties": { + "uses": { + "type": "string" + }, + "with": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "retry": { + "$ref": "#/$defs/RetryDefinition", + "default": { + "maxAttempts": 1, + "backoffMs": 0 + } + }, + "timeoutSeconds": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0, + "default": null + } + }, + "additionalProperties": false, + "required": [ + "uses" + ] + }, + "CompensationPolicyDefinition": { + "type": "object", + "properties": { + "onFailure": { + "$ref": "#/$defs/CompensationTrigger", + "default": "manual" + }, + "approval": { + "$ref": "#/$defs/ApprovalRequirement", + "default": "policy" + } + }, + "additionalProperties": false + }, + "CompensationTrigger": { + "type": "string", + "enum": [ + "manual", + "automatic" + ] + }, "PolicyDefinition": { "type": "object", "properties": { diff --git a/xtask/src/acceptance.rs b/xtask/src/acceptance.rs index 78bfd87..7eab3a2 100644 --- a/xtask/src/acceptance.rs +++ b/xtask/src/acceptance.rs @@ -17,7 +17,7 @@ use crate::process::{bounded_output, bounded_wait, configure_piped_command, outp const VERIFY_TOKEN: &str = "AGENTCTL_MOCK_FIXTURE_VERIFIED"; const LIVE_VERIFY_TOKEN: &str = "AGENTCTL_LIVE_FIXTURE_VERIFIED"; -const ACCEPTANCE_SCENARIOS: usize = 37; +const ACCEPTANCE_SCENARIOS: usize = 38; pub fn run(root: &Path) -> Result<()> { command(root, "cargo", &["build", "-p", "agentctl-cli", "--locked"])?; @@ -1617,6 +1617,109 @@ pub fn run(root: &Path) -> Result<()> { inspect(&binary, root, &subworkflow_db, subworkflow_replay_id)?; ensure!(array_len(&subworkflow_replay_inspect, "/data/effects")? == 0); + scenario( + 38, + "packaged CLI plans, executes, inspects, and replays compensation", + ); + let compensation_workflow = workspace.join("compensation.yaml"); + write(&compensation_workflow, COMPENSATION_WORKFLOW)?; + let compensation_db = directory.path().join("compensation.db"); + let source = json_with_code( + &binary, + root, + &run_args(&compensation_workflow, &compensation_db, &workspace, &[]), + 4, + )?; + let source_run_id = string_at(&source, "/error/runId")?; + let compensation_plan = successful_json( + &binary, + root, + &strings([ + "compensate", + source_run_id, + "--plan", + "--db", + path(&compensation_db)?, + "--workspace", + path(&workspace)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&compensation_plan, "/data/executable", true)?; + ensure_eq( + &compensation_plan, + "/data/tasks/0/sourceTaskId", + "provision", + )?; + let compensation = successful_json( + &binary, + root, + &strings([ + "compensate", + source_run_id, + "--db", + path(&compensation_db)?, + "--workspace", + path(&workspace)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&compensation, "/data/state", "succeeded")?; + ensure_eq(&compensation, "/data/compensatedTasks/0", "provision")?; + ensure!(fs::read_to_string(workspace.join("artifacts/compensation.txt"))? == "compensated"); + let compensation_run_id = string_at(&compensation, "/data/runId")?; + let compensation_inspect = inspect(&binary, root, &compensation_db, compensation_run_id)?; + ensure_eq( + &compensation_inspect, + "/data/run/sourceRunId", + source_run_id, + )?; + ensure_eq(&compensation_inspect, "/data/run/mode", "compensation")?; + let source_inspect = inspect(&binary, root, &compensation_db, source_run_id)?; + ensure_eq( + &source_inspect, + "/data/effectReconciliations/0/status", + "compensated", + )?; + let compensation_replay = successful_json( + &binary, + root, + &strings([ + "replay", + compensation_run_id, + "--db", + path(&compensation_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&compensation_replay, "/data/state", "succeeded")?; + let repeat_plan = successful_json( + &binary, + root, + &strings([ + "compensate", + source_run_id, + "--plan", + "--db", + path(&compensation_db)?, + "--output", + "json", + "--color", + "never", + ]), + )?; + ensure_eq(&repeat_plan, "/data/executable", false)?; + ensure!(array_len(&repeat_plan, "/data/alreadyCompensatedEffects")? == 1); + println!("agentctl credential-free acceptance passed ({ACCEPTANCE_SCENARIOS} scenarios)"); Ok(()) } @@ -3639,6 +3742,34 @@ spec: - { id: work, uses: "action:assign", with: { recovered: true } } "#; +const COMPENSATION_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 +kind: Workflow +metadata: { name: compensation-acceptance } +spec: + policy: + workspaceRoot: . + writableRoots: [artifacts] + approval: never + actions: + write: { kind: builtin.write } + assert: { kind: builtin.assert } + tasks: + - id: provision + uses: action:write + with: + path: artifacts/compensation.txt + content: provisioned + compensate: + uses: action:write + with: + path: artifacts/compensation.txt + content: compensated + - id: fail + uses: action:assert + needs: [provision] + with: { that: false, message: expected acceptance failure } +"#; + const CONTAINER_MOCK_WORKFLOW: &str = r#"apiVersion: agentctl.dev/v1alpha1 kind: Workflow metadata: { name: container-mock } From 63d1a721dfc43224665e7c2b48abd1e5ef96abfc Mon Sep 17 00:00:00 2001 From: Ompragash Date: Sat, 25 Jul 2026 14:07:51 +0530 Subject: [PATCH 17/44] feat: define structured role handoffs --- README.md | 1 + crates/agentctl-core/src/compiler.rs | 34 +++++ docs/ARCHITECTURE.md | 6 +- docs/COMPATIBILITY.md | 6 +- docs/DSL.md | 5 + docs/LIMITATIONS.md | 5 +- .../0014-structured-handoffs-as-graph-data.md | 35 +++++ docs/execution/COMPATIBILITY.md | 5 +- docs/execution/COMPLETENESS_VERIFICATION.md | 3 +- docs/execution/DECISIONS.md | 1 + docs/execution/EXAMPLE_VERIFICATION_MATRIX.md | 1 + docs/execution/LIMITATION_BURNDOWN.md | 46 +++--- docs/generated/CLI.md | 24 ++++ docs/guides/STRUCTURED_HANDOFFS.md | 67 +++++++++ docs/reference/TERMINOLOGY.md | 2 + examples/v1/README.md | 1 + examples/v1/fixture/team-evidence.txt | 1 + examples/v1/structured-handoff.yaml | 136 ++++++++++++++++++ xtask/src/acceptance.rs | 105 +++++++++++++- xtask/src/main.rs | 2 + 20 files changed, 461 insertions(+), 25 deletions(-) create mode 100644 docs/adr/0014-structured-handoffs-as-graph-data.md create mode 100644 docs/guides/STRUCTURED_HANDOFFS.md create mode 100644 examples/v1/fixture/team-evidence.txt create mode 100644 examples/v1/structured-handoff.yaml diff --git a/README.md b/README.md index 64aa672..333ce49 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ agentctl compensate SOURCE_RUN_ID ``` See [Retry a terminal workflow](docs/guides/TERMINAL_RETRY.md), [Repair a failed workflow](docs/guides/repair-a-failed-workflow.md), and [Compensate applied effects](docs/guides/COMPENSATION.md) for compatibility, lineage, state reconstruction, and uncertain-effect handling. +Use [Structured role handoffs](docs/guides/STRUCTURED_HANDOFFS.md) for bounded multi-role workflows without hidden conversation state. For retained pre-schema-5 history, use [Legacy run upgrade](docs/guides/LEGACY_RUN_UPGRADE.md). For ambiguous external outcomes, use [Effect reconciliation](docs/guides/EFFECT_RECONCILIATION.md). For confidential workflow history, use [Sensitive-state encryption](docs/guides/SENSITIVE_STATE_ENCRYPTION.md). For environment, mounted-file, and policy-gated process credentials, use [Secret references](docs/guides/SECRET_REFERENCES.md). diff --git a/crates/agentctl-core/src/compiler.rs b/crates/agentctl-core/src/compiler.rs index bf4cb4e..fcb47c2 100644 --- a/crates/agentctl-core/src/compiler.rs +++ b/crates/agentctl-core/src/compiler.rs @@ -1113,6 +1113,20 @@ pub fn compile(workflow: &Workflow, file: &str) -> Result