diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a26fc65..176521d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,10 @@ name: Rust -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] permissions: contents: read jobs: check: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -13,3 +13,17 @@ jobs: - run: cargo fmt --check - run: cargo clippy --all-targets --locked -- -D warnings - run: cargo test --locked + - name: Install sandbox runtime + run: sudo apt-get update && sudo apt-get install -y bubblewrap + - name: Verify real sandbox + env: + SPOOL_SECRET: must-not-reach-the-sandbox + run: cargo test --locked --test sandbox -- --ignored + - name: Build release candidate + run: bash scripts/package.sh + - uses: actions/upload-artifact@v4 + with: + name: spool-v0.0.1-linux-x86_64 + path: | + dist/*.tar.gz + dist/*.sha256 diff --git a/.gitignore b/.gitignore index cbf4253..1033423 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target/ /.spool/ +/dist/ diff --git a/AGENTS.md b/AGENTS.md index 0094c24..f35682b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ Build a small, embeddable Rust execution foundation for agents. Preserve useful - README describes shipped behaviour separately from proposed behaviour. - Remain in 0.0.n while iterating. Do not publish releases or claim production readiness as part of scaffolding. -- The current journal is not a sandbox, executor, scheduler or automatic recovery engine. +- The local preview executes explicit commands through bubblewrap. It is not yet an agent loop, MCP host, distributed scheduler or general security boundary for hostile workloads. - Do not expand into multi-agent scheduling, distributed workers or universal Bash/TypeScript compilation before local recovery is proven. - Keep Core integration optional: a workspace must be useful without a decision corpus. diff --git a/Cargo.toml b/Cargo.toml index cf290bd..7df3ff1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,10 @@ name = "asdecided-spool" version = "0.0.1" edition = "2024" +rust-version = "1.89" publish = false -description = "Durable execution workspace for AI agents — foundation prototype" +license = "Apache-2.0" +description = "Durable execution workspace for AI agents — local execution preview" [[bin]] name = "spool" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 285af5a..5ec0c02 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,87 @@ **Pick up where you left off.** -Spool is being built as a durable execution workspace for AI agents. Its goal is to keep working files, intermediate results and action history together so interrupted work can continue with an honest account of what completed and what remains uncertain. +Spool gives local command workflows a persistent workspace and an honest record of what completed. Queue commands, keep their files and output, and inspect interrupted work before continuing. -CLI and MCP are tool interfaces. Spool will provide the execution environment around them. AsDecided Core supplies versioned engineering decisions; Spool will record which decisions informed a job and surface relevant changes when it resumes. +**v0.0.1 preview:** Linux CLI and Rust library. Explicit commands run offline through bubblewrap. This is an execution foundation, not yet a model-driven agent or MCP host. AsDecided Core integration is planned and optional. -## Current status +Built for Omarchy — App -Foundation prototype, version 0.0.1 in source. No release has been published. +Omarchy is a target platform; this badge is a community identity label. No live Omarchy version has been tested yet. -Implemented: a Rust library and `spool` CLI backed by SQLite, with job creation, listing, inspection and append-only JSON checkpoints. Checkpoints survive closing and reopening the process. +## Install on your Omarchy XPS -**Not implemented yet:** tool execution, sandboxing, model integration, automatic resume, MCP adapters or AsDecided integration. A saved checkpoint is not a guarantee of safe execution recovery. +From a checkout of this candidate (Rust 1.89+): -## Try the foundation +```bash +sudo pacman -S --needed rust base-devel bubblewrap git +cargo install --path . --locked +export PATH="$HOME/.cargo/bin:$PATH" +spool --version +spool doctor +``` -Requires Rust with edition 2024 support and a C compiler for bundled SQLite. +`doctor` must pass before trying execution. Spool does not fall back to unsandboxed commands. Run it as your normal user, never with sudo. + +For a fresh checkout before publication, fetch the candidate branch: + +```bash +git clone --branch feat/v0.0.1-recovery https://github.com/asdecided/spool.git +cd spool +``` + +Once v0.0.1 is published, use `--branch v0.0.1` instead. Source installation avoids prebuilt glibc compatibility assumptions. CI also produces an x86_64 Linux archive and checksum for release preparation. + +## Try a persistent job + +In a new empty directory: ```bash -cargo build --locked -cargo install --path . --locked spool init -spool new "Investigate a failing build" -# Use the job ID returned above (1 in a fresh workspace). -spool checkpoint 1 '{"next_step":"inspect logs","logs_saved":true}' +spool new "XPS smoke test" +# Use the returned job ID; these examples assume 1. +spool queue 1 -- sh -c 'printf "hello from Spool\n" > hello.txt' +spool queue 1 -- cat hello.txt +spool resume 1 spool inspect 1 -spool status +# Completed operations are skipped; this does not run them again. +spool resume 1 ``` -State lives in `.spool/journal.sqlite3` under the current directory. Run commands from the same directory. This prototype stores checkpoint data in plaintext; keep credentials out of checkpoints. The journal is local metadata, not an isolated job workspace yet. +Files remain in `.spool/workspaces/1/`, mounted as `/work` in the sandbox. Your repository, home directory, credentials and journal are not mounted. System tools under `/usr` are read-only. Each command has a fresh temporary directory and environment; files in `/work` persist. Host networking is unavailable. + +## Recovery and limits + +- `spool exec 1 -- command args...` queues a command and runs pending work in order. +- `spool checkpoint 1 '{"next":"review"}'` retains explicit JSON state. +- `spool recover` marks interrupted dispatched operations **uncertain**. It never replays them. +- `spool resolve OP_ID succeeded 'evidence from inspecting the workspace'` records a manual conclusion. Use `failed` if the operation did not complete successfully. Do not guess. +- `spool events OP_ID` shows the operation history. +- Failed or uncertain operations block that job. After a failure, use a new job with a corrected command; no automatic retries are performed. +- `spool status` lists jobs; `spool inspect JOB_ID` shows checkpoints, operations and output. + +Only one process can own a journal, including readers. While a command runs, other commands against that journal report busy. Ctrl+C or terminating Spool leaves dispatched work for recovery. Bubblewrap is configured to die with its parent; inspect the workspace and ensure work has stopped before resolving uncertainty. + +Default timeout is 60 seconds. Set `SPOOL_TIMEOUT_SECONDS` to 1–3600. A timeout is uncertain because a command may have partially written files. Captured stdout and stderr are each capped at 64 KiB and excess bytes drained; truncation is reported. Workspace disk use, memory use and process count are **not quota-limited** in this preview. Use trusted commands and disposable workspaces, not hostile workloads. -## Direction +Journal/checkpoint/output data is plaintext. Do not place secrets in command arguments or checkpoints. Persisted files and external effects are not rolled back. This preview does not claim exactly-once execution or power-loss recovery of arbitrary files. Do not delete the journal lock file while Spool is running. -See [architecture](docs/architecture.md), [milestones](docs/roadmap.md) and [contributor guidance](AGENTS.md). +## Build and test ```bash cargo fmt --check cargo clippy --all-targets --locked -- -D warnings cargo test --locked +# Requires working Linux user/network namespaces and bubblewrap: +cargo test --locked --test sandbox -- --ignored +# Run after installing the candidate: +bash scripts/xps-smoke.sh ``` + +[Architecture](docs/architecture.md) · [Roadmap](docs/roadmap.md) · [XPS acceptance and release checklist](docs/v0.0.1.md) · [Verification evidence](docs/verification.md) + +## Uninstall and data + +`cargo uninstall asdecided-spool` removes a Cargo-installed executable. A manually installed archive binary can be removed from `~/.local/bin/spool`. Neither removes job data. Keep `.spool/` to preserve your work; copy it only while no Spool command is running. Schema 1 reads and migrates the initial unversioned journal. Older binaries must not be used after migration. + +Apache-2.0; see [LICENSE](LICENSE). diff --git a/docs/architecture.md b/docs/architecture.md index 8992e27..9f1f074 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,4 +30,4 @@ Engineering decisions guide planning and review. Only explicit executable polici ## Current implementation -Only jobs and JSON checkpoints exist. The schema is bootstrapped on open; introduce versioned migrations before changing the persisted schema. SQLite uses foreign keys, WAL and FULL synchronous mode. This is a local prototype with no encryption, hostile-client isolation or arbitrary-code execution. +Schema 1 adds operations and append-only transition events to the initial jobs/checkpoints schema. Migration is transactional and newer schema versions are rejected. SQLite uses foreign keys, WAL and FULL synchronous mode. An OS file lock serialises journal owners and releases on process death. Explicit commands execute through bubblewrap with an offline per-job workspace, a wall-clock timeout and bounded captured output. Recovery marks dispatched actions uncertain and manual resolution records evidence. The preview has no model loop, MCP adapter, encryption, quotas on workspace disk/memory/process use, or claim of hostile-workload isolation. diff --git a/docs/roadmap.md b/docs/roadmap.md index 147240e..3929304 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,11 +4,11 @@ Rust library and CLI; SQLite jobs; append-only JSON checkpoints; reopening and validation tests; CI. This milestone proves basic persistence, not crash-safe tool execution. -## 1. Operation journal and recovery +## 1. Operation journal and recovery (implemented in v0.0.1 candidate) Versioned schema migrations, operation state machine, atomic worker ownership, fake executor and explicit reconciliation. Test crashes before dispatch, during dispatch and after an external effect but before local completion. Verify uncertain effects are never automatically repeated. Prove recovery in separate processes. -## 2. Restricted local execution +## 2. Restricted local execution (initial candidate implementation; live acceptance pending) Per-job workspaces, artifact storage, bounded outputs, timeouts and cancellation, actual platform sandboxing, secret references and permission checks. Start with one job/worker. Test surviving child processes and paths escaping the workspace. Unsupported platforms must fail closed for operations requiring sandboxing. diff --git a/docs/v0.0.1.md b/docs/v0.0.1.md new file mode 100644 index 0000000..e944ae2 --- /dev/null +++ b/docs/v0.0.1.md @@ -0,0 +1,46 @@ +# v0.0.1 acceptance and release candidate + +## Purpose + +Test a small local durable execution foundation on Tom's Omarchy XPS. No model account or API key is required. This version executes explicit offline commands, keeps their workspace and output, and prevents blind replay after an interruption. + +## XPS acceptance + +1. Record `omarchy-version`, `uname -a`, `rustc --version`, and `bwrap --version`. +2. Install from source using the README. Run `spool doctor` as your normal user. A namespace error is a blocker, not permission to run unsandboxed. +3. Run `bash scripts/xps-smoke.sh`. It prints the retained temporary directory. Check that `proof.txt` contains exactly `hello`, including after a second resume. +4. Inspect the timed-out job. Its state must be uncertain and a second resume must refuse it. +5. In another disposable directory create a job, queue `sh -c 'printf once >> proof.txt; sleep 60'` and resume it. While it is running, a second terminal's `spool status` in that same directory must report busy. Stop the first process with Ctrl+C. Run `spool recover`, inspect the job, and verify that `proof.txt` is not appended again. Confirm the sandbox process is gone before resolving the result. +6. Run the real sandbox integration test: `cargo test --locked --test sandbox -- --ignored`. + +Keep this testing separate from valuable work. Never put secrets in arguments, checkpoints or output. Tests do not need access to your source tree or home directory. + +## Installation from the CI archive + +Download `spool-v0.0.1-linux-x86_64.tar.gz` and its `.sha256` from the candidate's GitHub Actions artifact (or published release later). In their download directory: + +```bash +sha256sum -c spool-v0.0.1-linux-x86_64.tar.gz.sha256 +tar -xzf spool-v0.0.1-linux-x86_64.tar.gz +mkdir -p "$HOME/.local/bin" +install -m755 spool-v0.0.1-linux-x86_64/spool "$HOME/.local/bin/spool" +export PATH="$HOME/.local/bin:$PATH" +spool --version +spool doctor +``` + +Runtime dependency: bubblewrap at `/usr/bin/bwrap`; Linux with enabled unprivileged namespaces. The executable uses bundled SQLite but dynamically links glibc. Only x86_64 Linux is packaged. Use source installation if the binary is incompatible. Remove `~/.local/bin/spool` to uninstall; preserve `.spool/` directories. + +## Publication gate + +- Required GitHub CI and sandbox test green on the exact candidate commit. +- Review the PR and confirm no unexpected changes. +- Record XPS acceptance results; if publishing before XPS acceptance, label that limitation explicitly. +- Merge and create tag `v0.0.1` only when authorized. This candidate does not create a release automatically. +- Attach the matching CI archive and checksum to a prerelease, along with the source commit and known limitations. + +## Suggested release notes + +Spool v0.0.1 is a local Linux execution preview. Queue explicit commands, keep their workspace between runs, and inspect what happened after an interruption. Completed operations are skipped; uncertain ones stop for review. + +This first version includes a SQLite operation journal, checkpoint history, exclusive workspace ownership, schema migration and offline bubblewrap execution with timeouts and capped captured output. It has no model loop, MCP integration or AsDecided decision integration yet. There are no workspace disk, memory or process-count quotas; use trusted commands and disposable workspaces. diff --git a/docs/verification-inputs.json b/docs/verification-inputs.json new file mode 100644 index 0000000..86ab709 --- /dev/null +++ b/docs/verification-inputs.json @@ -0,0 +1,10 @@ +{ + "Cargo.toml": "66d4266efaf799a87e312c080325b2148f5ccb22f7ae833bc9357f2e00336369", + "Cargo.lock": "ee1121a9a7646aa0c5b3ddb497e0d318fc2f223dff3114d8b319a32f0dd8fcb6", + "src/lib.rs": "96855a1b2101d18b92b6aa75cd9e3139d484651af8b0b88074fda836697f2918", + "src/main.rs": "e258fa6ae046ddd3538775af8bd127520c6e81d79489b5213f980ccfe9d4dc8c", + "src/execution.rs": "ded548f3acd37dcdda6c3b1367822f247e54a1e6ad3423094a73d0a162f6e6ce", + "tests/cli.rs": "844c5ac545c318cdadb7749181ac1f7fbbc90b157c01224ba240a5d2591ea828", + "tests/journal.rs": "3e6eb3b4010b5a95f69529966edb92130df2eb2c08abaddbf8032847782b9571", + "tests/sandbox.rs": "8ab3f2c07798916e45646b8dd5efcf40ec5c4a6ab61f1775e0395d742f34ecd5" +} diff --git a/docs/verification.md b/docs/verification.md new file mode 100644 index 0000000..a9edd0a --- /dev/null +++ b/docs/verification.md @@ -0,0 +1,25 @@ +# Candidate verification + +Inputs are identified by SHA-256 in [verification-inputs.json](verification-inputs.json). These hashes identify the tested code, not a promise of compatibility. + +## Reproduced locally + +Environment: x86_64 Ubuntu 24.04 container, Rust 1.98.1, Cargo 1.98.1. + +- `cargo fmt --check`: pass. +- `cargo clippy --all-targets --locked -- -D warnings`: pass. +- `cargo test --locked`: seven ordinary tests pass. The ignored crash-worker helper is invoked and killed by the parent recovery test at four interruption points. +- Recovery test: second process denied ownership; lock released on process kill; interrupted dispatch becomes uncertain; completed work remains completed; repeated recovery changes nothing; uncertain work refuses resume; evidence-backed manual resolution retained. +- Legacy unversioned journal migration retains checkpoints. Newer schema is refused. +- Output reader retains at most 64 KiB per stream and drains excess. +- `bash -n scripts/package.sh scripts/xps-smoke.sh`: pass. + +## Blocked locally + +`spool doctor` / real bubblewrap execution: container policy prevents creation of the network namespace (`NETLINK_ROUTE socket: Operation not permitted`). No restrictions were relaxed. The opt-in real sandbox test is a required CI step on Ubuntu 22.04, not counted as locally passed. + +## Not run locally + +Live Omarchy XPS installation, Arch package-manager installation, actual Ctrl+C recovery of a sandbox workload, filesystem power-loss tests, hostile-workload isolation, and memory/disk/process quota tests. This release has no such quotas. Native Arch packaging is not included; install from source or use the CI archive. + +The GitHub PR carries current CI results for the exact commit. Check those before publication. Follow [v0.0.1 acceptance](v0.0.1.md) on the XPS. diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100644 index 0000000..d094ffa --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +test "$(uname -m)" = x86_64 +test "$(uname -s)" = Linux +cargo build --release --locked +package=spool-v0.0.1-linux-x86_64 +mkdir -p "dist/$package" +install -m755 target/release/spool "dist/$package/spool" +install -m644 README.md LICENSE docs/v0.0.1.md "dist/$package/" +tar -C dist -czf "dist/$package.tar.gz" "$package" +(cd dist && sha256sum "$package.tar.gz" > "$package.tar.gz.sha256") +printf 'Prepared %s\n' "dist/$package.tar.gz" diff --git a/scripts/xps-smoke.sh b/scripts/xps-smoke.sh new file mode 100644 index 0000000..69b7b66 --- /dev/null +++ b/scripts/xps-smoke.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +spool --version +spool doctor +test_dir=$(mktemp -d "${TMPDIR:-/tmp}/spool-smoke.XXXXXX") +printf 'Test workspace retained at %s\n' "$test_dir" +cd "$test_dir" +spool init +job=$(spool new 'XPS smoke') +spool queue "$job" -- sh -c 'printf hello >> proof.txt' +spool queue "$job" -- cat proof.txt +spool resume "$job" +spool resume "$job" +test "$(cat ".spool/workspaces/$job/proof.txt")" = hello +spool checkpoint "$job" '{"smoke":"passed"}' +spool inspect "$job" +slow=$(spool new 'Timeout should stop recovery') +spool queue "$slow" -- sh -c 'printf started > partial.txt; sleep 10' +if SPOOL_TIMEOUT_SECONDS=1 spool resume "$slow"; then + printf 'ERROR: slow operation unexpectedly succeeded\n' >&2 + exit 1 +fi +if spool resume "$slow"; then + printf 'ERROR: uncertain operation was replayed\n' >&2 + exit 1 +fi +spool inspect "$slow" +printf 'PASS: persistent files, no replay of completed work, and blocked uncertain work.\n' diff --git a/src/execution.rs b/src/execution.rs new file mode 100644 index 0000000..c032017 --- /dev/null +++ b/src/execution.rs @@ -0,0 +1,384 @@ +use crate::{Journal, Result}; +use rusqlite::params; +use serde_json::{Value, json}; +use std::io::Read; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +const OUTPUT_LIMIT: usize = 64 * 1024; + +impl Journal { + pub fn queue(&self, job: i64, argv: &[String]) -> Result { + if argv.is_empty() || argv[0].is_empty() { + return Err("command must not be empty".into()); + } + let tx = self.0.unchecked_transaction()?; + tx.execute( + "INSERT INTO operations(job_id, argv, state) VALUES (?1, ?2, 'prepared')", + params![job, serde_json::to_string(argv)?], + )?; + let id = tx.last_insert_rowid(); + tx.execute( + "INSERT INTO events(operation_id, state, evidence) VALUES (?1, 'prepared', '{}')", + [id], + )?; + tx.commit()?; + Ok(id) + } + + pub fn operations(&self, job: i64) -> Result> { + let mut stmt = self.0.prepare( + "SELECT id, argv, state, result FROM operations WHERE job_id=?1 ORDER BY id", + )?; + let rows = stmt.query_map([job], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, Option>(3)?, + )) + })?; + rows.map(|row| { + let (id, argv, state, result) = row?; + Ok( + json!({"id":id,"argv":serde_json::from_str::(&argv)?,"state":state, + "result":result.map(|s|serde_json::from_str::(&s)).transpose()?}), + ) + }) + .collect() + } + + /// Atomically records a state transition and its evidence. + fn transition(&self, id: i64, from: &str, to: &str, evidence: Value) -> Result<()> { + let tx = self.0.unchecked_transaction()?; + let changed = tx.execute( + "UPDATE operations SET state=?1, result=?2 WHERE id=?3 AND state=?4", + params![to, evidence.to_string(), id, from], + )?; + if changed != 1 { + return Err("operation is missing or no longer in the required state".into()); + } + tx.execute( + "INSERT INTO events(operation_id, state, evidence) VALUES (?1, ?2, ?3)", + params![id, to, evidence.to_string()], + )?; + tx.commit()?; + Ok(()) + } + + /// Only the journal owner can recover. Never replays work. + pub fn recover(&self) -> Result { + let tx = self.0.unchecked_transaction()?; + tx.execute("INSERT INTO events(operation_id,state,evidence) SELECT id,'uncertain','{\"reason\":\"owner interrupted before recording outcome\"}' FROM operations WHERE state='dispatched'", [])?; + let count = tx.execute("UPDATE operations SET state='uncertain',result='{\"reason\":\"owner interrupted before recording outcome\"}' WHERE state='dispatched'", [])?; + tx.commit()?; + Ok(count) + } + + pub fn resolve(&self, id: i64, outcome: &str, note: &str) -> Result<()> { + if !["succeeded", "failed"].contains(&outcome) || note.trim().is_empty() { + return Err("resolve requires succeeded|failed and a nonempty evidence note".into()); + } + self.transition(id, "uncertain", outcome, json!({"manual_resolution":note})) + } + + pub fn events(&self, id: i64) -> Result> { + let mut stmt = self.0.prepare( + "SELECT state,evidence,created_at FROM events WHERE operation_id=?1 ORDER BY id", + )?; + let rows = stmt.query_map([id], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })?; + rows.map(|row| { + let (state, evidence, at) = row?; + Ok(json!({"state":state,"evidence":serde_json::from_str::(&evidence)?,"at":at})) + }) + .collect() + } + + /// Execute prepared operations in order, stopping at failures or uncertainty. + pub fn resume(&self, job: i64, root: &Path, timeout: Duration) -> Result<()> { + self.inspect(job)?; + self.recover()?; + let operations = self.operations(job)?; + if operations + .iter() + .any(|op| ["uncertain", "failed"].contains(&op["state"].as_str().unwrap_or(""))) + { + return Err("job contains failed or uncertain operations; inspect it before continuing (create a new job after a failure)".into()); + } + for op in operations { + if op["state"] != "prepared" { + continue; + } + let argv: Vec = serde_json::from_value(op["argv"].clone())?; + let id = op["id"].as_i64().ok_or("invalid operation id")?; + let workspace = root.join("workspaces").join(job.to_string()); + std::fs::create_dir_all(&workspace)?; + let mut command = sandbox_command(&workspace, &argv)?; + self.transition( + id, + "prepared", + "dispatched", + json!({"timeout_seconds":timeout.as_secs()}), + )?; + let outcome = capture(&mut command, timeout); + match outcome { + Ok(result) => { + // A nonzero exit may have made partial changes. It is never retried automatically. + let state = if result["timed_out"] == true { + "uncertain" + } else if result["exit_code"] == 0 { + "succeeded" + } else { + "failed" + }; + self.transition(id, "dispatched", state, result)?; + if state != "succeeded" { + return Err( + format!("operation {id} {state}; inspect the job for output").into(), + ); + } + } + Err(error) => { + self.transition( + id, + "dispatched", + "uncertain", + json!({"error":error.to_string()}), + )?; + return Err(error); + } + } + } + Ok(()) + } +} + +fn sandbox_command(workspace: &Path, argv: &[String]) -> Result { + if !cfg!(target_os = "linux") { + return Err("execution requires Linux and bubblewrap".into()); + } + let mut command = Command::new("/usr/bin/bwrap"); + command.env_clear().args([ + "--unshare-all", + "--die-with-parent", + "--new-session", + "--cap-drop", + "ALL", + "--ro-bind", + "/usr", + "/usr", + "--symlink", + "usr/bin", + "/bin", + "--symlink", + "usr/bin", + "/sbin", + ]); + for path in ["/lib", "/lib64"] { + if Path::new(path).exists() { + command.args(["--ro-bind", path, path]); + } + } + command + .args([ + "--proc", + "/proc", + "--dev", + "/dev", + "--tmpfs", + "/tmp", + "--dir", + "/home", + "--setenv", + "HOME", + "/home", + "--setenv", + "PATH", + "/usr/bin:/bin", + "--setenv", + "LANG", + "C.UTF-8", + "--bind", + ]) + .arg(workspace.canonicalize()?) + .args(["/work", "--chdir", "/work", "--"]) + .args(argv); + Ok(command) +} + +fn read_bounded(mut reader: impl Read) -> std::io::Result<(Vec, bool)> { + let mut output = Vec::new(); + let mut buffer = [0u8; 8192]; + let mut truncated = false; + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + let keep = n.min(OUTPUT_LIMIT - output.len()); + output.extend_from_slice(&buffer[..keep]); + truncated |= keep < n; + } + Ok((output, truncated)) +} + +fn capture(command: &mut Command, timeout: Duration) -> Result { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout = child.stdout.take().ok_or("missing stdout")?; + let stderr = child.stderr.take().ok_or("missing stderr")?; + let out = std::thread::spawn(move || read_bounded(stdout)); + let err = std::thread::spawn(move || read_bounded(stderr)); + let start = Instant::now(); + let mut timed_out = false; + let status = loop { + if let Some(status) = child.try_wait()? { + break status; + } + if start.elapsed() >= timeout { + timed_out = true; + child.kill()?; + break child.wait()?; + } + std::thread::sleep(Duration::from_millis(20)); + }; + let (stdout, out_truncated) = out.join().map_err(|_| "stdout reader panicked")??; + let (stderr, err_truncated) = err.join().map_err(|_| "stderr reader panicked")??; + Ok( + json!({"exit_code":status.code(),"timed_out":timed_out,"stdout":String::from_utf8_lossy(&stdout), + "stderr":String::from_utf8_lossy(&stderr),"stdout_truncated":out_truncated,"stderr_truncated":err_truncated}), + ) +} + +pub fn doctor() -> Result { + let workspace = std::env::current_dir()?; + let mut cmd = sandbox_command(&workspace, &["/bin/true".to_string()])?; + let result = capture(&mut cmd, Duration::from_secs(5))?; + if result["exit_code"] != 0 { + return Err(format!("sandbox unavailable: {}", result["stderr"]).into()); + } + Ok(json!({"sandbox":"bubblewrap","available":true,"network":false})) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn output_is_bounded_and_drained() { + let data = vec![b'x'; OUTPUT_LIMIT + 500]; + let (bytes, truncated) = read_bounded(&data[..]).unwrap(); + assert_eq!(bytes.len(), OUTPUT_LIMIT); + assert!(truncated); + } + + // This helper exists only in the test executable, never the shipped CLI. + #[test] + #[ignore] + fn crash_worker() { + let root = std::env::var("SPOOL_TEST_ROOT").unwrap(); + let phase = std::env::var("SPOOL_TEST_PHASE").unwrap(); + let path = Path::new(&root); + let journal = Journal::open(&path.join("journal.sqlite3")).unwrap(); + let job = journal.create("Crash test").unwrap(); + let op = journal.queue(job, &["fake-effect".into()]).unwrap(); + if phase != "prepared" { + journal + .transition(op, "prepared", "dispatched", json!({})) + .unwrap(); + } + if phase == "effect" || phase == "completed" { + let mut file = std::fs::File::create(path.join("external-effect")).unwrap(); + std::io::Write::write_all(&mut file, b"once").unwrap(); + file.sync_all().unwrap(); + } + if phase == "completed" { + journal + .transition(op, "dispatched", "succeeded", json!({"receipt":"once"})) + .unwrap(); + } + std::fs::write(path.join("ready"), b"ready").unwrap(); + loop { + std::thread::park(); + } + } + + #[test] + fn killed_workers_preserve_outcomes_and_never_replay_uncertainty() { + for phase in ["prepared", "dispatched", "effect", "completed"] { + let dir = tempfile::tempdir().unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", "execution::tests::crash_worker"]) + .env("SPOOL_TEST_ROOT", dir.path()) + .env("SPOOL_TEST_PHASE", phase) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let start = Instant::now(); + while !dir.path().join("ready").exists() { + if start.elapsed() > Duration::from_secs(10) { + let _ = child.kill(); + let _ = child.wait(); + panic!("worker did not become ready"); + } + assert!(child.try_wait().unwrap().is_none(), "worker exited early"); + std::thread::sleep(Duration::from_millis(10)); + } + assert!(Journal::open(&dir.path().join("journal.sqlite3")).is_err()); + child.kill().unwrap(); + child.wait().unwrap(); + let journal = Journal::open(&dir.path().join("journal.sqlite3")).unwrap(); + let count = journal.recover().unwrap(); + let expected = match phase { + "prepared" => "prepared", + "completed" => "succeeded", + _ => "uncertain", + }; + assert_eq!(journal.operations(1).unwrap()[0]["state"], expected); + assert_eq!(count, usize::from(expected == "uncertain")); + assert_eq!(journal.recover().unwrap(), 0); + if expected == "uncertain" { + assert!( + journal + .resume(1, dir.path(), Duration::from_secs(1)) + .is_err() + ); + assert!( + journal + .transition(1, "prepared", "dispatched", json!({})) + .is_err() + ); + journal + .resolve( + 1, + "succeeded", + "Inspected fake destination; effect reconciled", + ) + .unwrap(); + assert!(journal.resolve(1, "succeeded", "again").is_err()); + assert_eq!( + journal.events(1).unwrap().last().unwrap()["state"], + "succeeded" + ); + } + if phase == "effect" || phase == "completed" { + assert_eq!( + std::fs::read(dir.path().join("external-effect")).unwrap(), + b"once" + ); + } else { + assert!(!dir.path().join("external-effect").exists()); + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 5498126..01b6bc8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,14 +1,37 @@ use rusqlite::{Connection, OptionalExtension, params}; +use std::fs::{File, OpenOptions}; use std::path::Path; +pub mod execution; + pub type Result = std::result::Result>; /// Local job metadata. This journal does not execute or replay tools. -pub struct Journal(Connection); +pub struct Journal(Connection, File); + +impl Drop for Journal { + fn drop(&mut self) { + let _ = self.1.unlock(); + } +} impl Journal { pub fn open(path: &Path) -> Result { - let db = Connection::open(path)?; + // The lock file must remain in place: unlinking it would split ownership. + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path.with_extension("lock"))?; + lock.try_lock() + .map_err(|_| "workspace busy: another Spool process owns this journal")?; + let mut db = Connection::open(path)?; + db.busy_timeout(std::time::Duration::from_secs(5))?; + let version: i64 = db.query_row("PRAGMA user_version", [], |r| r.get(0))?; + if version > 1 { + return Err("journal was created by a newer Spool; upgrade before opening it".into()); + } db.busy_timeout(std::time::Duration::from_secs(5))?; db.execute_batch( "PRAGMA foreign_keys = ON; @@ -26,7 +49,27 @@ impl Journal { created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP );", )?; - Ok(Self(db)) + if version == 0 { + let tx = db.transaction()?; + tx.execute_batch("CREATE TABLE operations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id INTEGER NOT NULL REFERENCES jobs(id), + argv TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('prepared','dispatched','succeeded','failed','uncertain')), + result TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation_id INTEGER NOT NULL REFERENCES operations(id), + state TEXT NOT NULL, + evidence TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + PRAGMA user_version = 1;")?; + tx.commit()?; + } + Ok(Self(db, lock)) } pub fn create(&self, goal: &str) -> Result { @@ -64,7 +107,9 @@ impl Journal { let checkpoint = payload .map(|p| serde_json::from_str::(&p)) .transpose()?; - Ok(serde_json::json!({"id": job, "goal": goal, "checkpoint": checkpoint})) + Ok( + serde_json::json!({"id": job, "goal": goal, "checkpoint": checkpoint, "operations": self.operations(job)?}), + ) } pub fn list(&self) -> Result> { diff --git a/src/main.rs b/src/main.rs index c024350..b4bde78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,10 +12,19 @@ fn run() -> Result<()> { let args: Vec = std::env::args().skip(1).collect(); if args.is_empty() || args == ["--help"] || args == ["help"] { println!( - "Spool — pick up where you left off.\nFoundation prototype: records jobs and JSON checkpoints only.\n\nspool init\nspool new \nspool status\nspool inspect \nspool checkpoint " + "Spool — pick up where you left off.\nLocal Linux execution preview v{}\n\nspool init\nspool doctor\nspool new \nspool status\nspool inspect \nspool checkpoint \nspool queue -- [args...]\nspool exec -- [args...]\nspool resume \nspool recover\nspool resolve succeeded|failed \nspool events \n\nCommands run offline inside a bubblewrap sandbox with a per-job /work directory.\nTimeout: 60 seconds; override with SPOOL_TIMEOUT_SECONDS (1..3600).", + env!("CARGO_PKG_VERSION") ); return Ok(()); } + if args == ["--version"] { + println!("spool {}", env!("CARGO_PKG_VERSION")); + return Ok(()); + } + if args == ["doctor"] { + println!("{}", asdecided_spool::execution::doctor()?); + return Ok(()); + } let path = Path::new(".spool/journal.sqlite3"); if args == ["init"] { std::fs::create_dir_all(".spool")?; @@ -39,7 +48,41 @@ fn run() -> Result<()> { "checkpoint" if args.len() == 3 => { println!("{}", journal.checkpoint(args[1].parse()?, &args[2])?) } + "queue" | "exec" if args.len() >= 4 && args[2] == "--" => { + let job = args[1].parse()?; + let timeout = timeout()?; + let operation = journal.queue(job, &args[3..])?; + println!("Queued operation {operation}"); + if args[0] == "exec" { + journal.resume(job, Path::new(".spool"), timeout)?; + println!("{}", journal.inspect(job)?); + } + } + "resume" if args.len() == 2 => { + let job = args[1].parse()?; + journal.resume(job, Path::new(".spool"), timeout()?)?; + println!("{}", journal.inspect(job)?); + } + "recover" if args.len() == 1 => println!( + "Marked {} interrupted operations uncertain; nothing replayed", + journal.recover()? + ), + "resolve" if args.len() == 4 => journal.resolve(args[1].parse()?, &args[2], &args[3])?, + "events" if args.len() == 2 => println!( + "{}", + serde_json::to_string_pretty(&journal.events(args[1].parse()?)?)? + ), _ => return Err("invalid arguments; see `spool --help`".into()), } Ok(()) } + +fn timeout() -> Result { + let seconds: u64 = std::env::var("SPOOL_TIMEOUT_SECONDS") + .unwrap_or_else(|_| "60".into()) + .parse()?; + if !(1..=3600).contains(&seconds) { + return Err("timeout must be 1..3600 seconds".into()); + } + Ok(std::time::Duration::from_secs(seconds)) +} diff --git a/tests/journal.rs b/tests/journal.rs index 0aa4faf..f997cae 100644 --- a/tests/journal.rs +++ b/tests/journal.rs @@ -31,3 +31,35 @@ fn invalid_updates_do_not_replace_the_last_checkpoint() { assert!(journal.inspect(job + 1).is_err()); assert_eq!(journal.inspect(job).unwrap()["checkpoint"]["ok"], true); } + +#[test] +fn legacy_database_migrates_without_losing_checkpoints() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("journal.sqlite3"); + { + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch("CREATE TABLE jobs(id INTEGER PRIMARY KEY AUTOINCREMENT,goal TEXT NOT NULL,created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); + CREATE TABLE checkpoints(id INTEGER PRIMARY KEY AUTOINCREMENT,job_id INTEGER NOT NULL REFERENCES jobs(id),payload TEXT NOT NULL,created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); + INSERT INTO jobs(goal) VALUES ('legacy'); + INSERT INTO checkpoints(job_id,payload) VALUES (1,'{\"saved\":true}');").unwrap(); + } + let journal = Journal::open(&path).unwrap(); + assert_eq!(journal.inspect(1).unwrap()["checkpoint"]["saved"], true); + journal.queue(1, &["true".into()]).unwrap(); + drop(journal); + let journal = Journal::open(&path).unwrap(); + assert_eq!(journal.operations(1).unwrap().len(), 1); +} + +#[test] +fn newer_schema_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("journal.sqlite3"); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch("PRAGMA user_version=99;").unwrap(); + assert!(Journal::open(&path).is_err()); + let version: i64 = db + .query_row("PRAGMA user_version", [], |r| r.get(0)) + .unwrap(); + assert_eq!(version, 99); +} diff --git a/tests/sandbox.rs b/tests/sandbox.rs new file mode 100644 index 0000000..a0a7cd5 --- /dev/null +++ b/tests/sandbox.rs @@ -0,0 +1,38 @@ +//! Explicit opt-in: restricted containers may not permit user/network namespaces. +use asdecided_spool::Journal; +use std::time::Duration; + +#[test] +#[ignore = "requires Linux bubblewrap with working unprivileged namespaces"] +fn real_sandbox_persists_files_hides_home_and_bounds_timeout() { + asdecided_spool::execution::doctor().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let journal = Journal::open(&dir.path().join("journal.sqlite3")).unwrap(); + let job = journal.create("sandbox acceptance").unwrap(); + journal.queue(job, &["sh".into(), "-c".into(), + "test ! -e /root && test ! -e /work/journal.sqlite3 && test -z \"$SPOOL_SECRET\" && printf hello > proof.txt && cat proof.txt".into()]).unwrap(); + journal + .resume(job, dir.path(), Duration::from_secs(5)) + .unwrap(); + assert_eq!( + journal.operations(job).unwrap()[0]["result"]["stdout"], + "hello" + ); + journal + .resume(job, dir.path(), Duration::from_secs(5)) + .unwrap(); + assert_eq!( + std::fs::read(dir.path().join("workspaces/1/proof.txt")).unwrap(), + b"hello" + ); + let slow = journal.create("timeout").unwrap(); + journal + .queue(slow, &["sh".into(), "-c".into(), "sleep 10".into()]) + .unwrap(); + assert!( + journal + .resume(slow, dir.path(), Duration::from_millis(100)) + .is_err() + ); + assert_eq!(journal.operations(slow).unwrap()[0]["state"], "uncertain"); +}