diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da4b396..ae2b6c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: - run: npm ci - run: npm run check - run: npm test + - run: node scripts/evaluate-retrieval.js - name: Smoke test packed CLI shell: bash run: | diff --git a/README.md b/README.md index df5ed9b..d8252d2 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ Context management for AI coding agents. [agenctx.com](https://agenctx.com) ## What is agenctx? +For integrations, see [versioned context and receipt exports](docs/export-format.md). +For search regression checks, see [retrieval evaluation](docs/retrieval-evaluation.md). + `agenctx` is a local, version-controlled context management system for AI agents. It gives each repository a shared memory for the rules, decisions, warnings, test requirements, design patterns, security constraints, and other knowledge an agent needs to work safely. Humans maintain that knowledge inside `.agenctx/`. Agents discover it through a small generated guide, search the relevant context channel, inspect previews, and open the exact entries they need before acting. The complete store is not copied into every prompt. diff --git a/bin/agenctx.js b/bin/agenctx.js index fb338e0..b732d18 100755 --- a/bin/agenctx.js +++ b/bin/agenctx.js @@ -24,6 +24,9 @@ Core commands: dump [target] Generate agent guides (openai/claude/cursor/all) dump [target] --check Check generated sections without changing files sync Re-scan project for package/env changes + export context Print a versioned JSON context snapshot + export receipt Print a verified sealed receipt as JSON + export validate Validate an export without importing it Lifecycle: lifecycle [--all] Inspect policy, states, decay, and archive @@ -242,6 +245,10 @@ async function main() { await search(args); break; } + case 'export': { + require('../lib/commands/export').exportCommand(args); + break; + } case 'session': { const { session } = require('../lib/commands/session'); session(args); diff --git a/docs/export-format.md b/docs/export-format.md new file mode 100644 index 0000000..4e5eb3d --- /dev/null +++ b/docs/export-format.md @@ -0,0 +1,47 @@ +# Versioned exports + +From an initialized repository: + +```sh +agenctx export context > context-export.json +agenctx export receipt > receipt-export.json +agenctx export validate context-export.json +``` + +In Windows PowerShell 5, save the UTF-8 output explicitly: + +```powershell +agenctx export context | Set-Content -Encoding UTF8 context-export.json +``` + +Validation accepts a UTF-8 BOM and works outside an initialized repository. +The export commands write only JSON to stdout; errors go to stderr with nonzero +exit status. Receipt selectors must identify exactly one sealed receipt. +The source-checkout consumer `node examples/read-export.js context-export.json` +demonstrates reading either kind without parsing terminal display output. + +The export envelope uses `format: "agenctx-export"`, `version: 1`, a `kind` +(`context` or `receipt`), and a `data` object. Unknown envelope versions and kinds +must be rejected rather than interpreted as the current format. New optional +fields within data may be preserved by consumers. + +A context export contains `config` and `state` objects, including all entry +statuses. It is a snapshot of maintained knowledge, not a selection of trusted +instructions for a particular task. It excludes proposals, history, runtime +sessions, credentials outside the store, and generated agent guides. Entries +have globally unique nonempty string IDs, string content, and one of `active`, +`pinned`, `ambient`, or `archived` statuses. Banner entries remain in source order. + +A receipt export contains one sealed receipt, including its original hash and +ordered served events. The export does not recompute timestamps or invent proof +that an agent followed the instructions. Consumers must verify its original +receipt hash. Legacy sessions without a verifiable hash cannot be exported as +verified receipts. + +Exports can contain private repository text; they stay local unless the user +chooses to transfer them. Export is read-only. Validation does not approve +imported context, and this format does not introduce automatic import or merge. + +Round-trip compatibility means `decode(encode(envelope))` preserves the JSON +values. It does not promise byte-for-byte equality with source files or preserve +formatting. Incompatible changes require a new version and migration guidance. diff --git a/docs/retrieval-evaluation.md b/docs/retrieval-evaluation.md new file mode 100644 index 0000000..68b6876 --- /dev/null +++ b/docs/retrieval-evaluation.md @@ -0,0 +1,22 @@ +# Retrieval regression evaluation + +Run `node scripts/evaluate-retrieval.js` from the checkout. It emits JSON and +exits nonzero when any expected result list changes. Supply another fixture +path as the first argument to evaluate additional cases. + +The version-1 fixture contains a fixed state and queries with explicit expected +entry IDs. Cases cover case-insensitive substring matching, banner/date filters, +multiline matches, no matches, and ambient/archived visibility. The unit suite +also checks exact-ID access to full text. The evaluator calls the same pure +selection functions as noninteractive search without changing stored state. + +Precision is the fraction of returned IDs that are relevant; recall is the +fraction of expected IDs returned. Empty expected and returned sets receive 1 +for both. An empty result against nonempty expectations receives zero. Exact +match additionally requires the expected order. Macro metrics weight cases +equally. The regression gate is 100% exact match on the checked-in cases. + +This is a synthetic mechanics suite, not a study of agent task completion, +retrieval relevance in arbitrary repositories, lifecycle scheduling, UI behavior, +or token savings. Add fixtures with explicit expected outcomes before changing +the baseline; do not regenerate expectations from the implementation. diff --git a/examples/read-export.js b/examples/read-export.js new file mode 100644 index 0000000..1dfdcd6 --- /dev/null +++ b/examples/read-export.js @@ -0,0 +1,14 @@ +'use strict'; +// Example consumer: no dependency on Agenctx's terminal formatting. +const fs = require('fs'); +const { decode } = require('../lib/export-format'); + +const envelope = decode(fs.readFileSync(process.argv[2], 'utf8')); +if (envelope.kind === 'context') { + const entries = Object.values(envelope.data.state.banners).flatMap(b => b.entries); + console.log(JSON.stringify({ kind: 'context', entries: entries.length, + statuses: [...new Set(entries.map(e => e.status))].sort() }, null, 2)); +} else { + console.log(JSON.stringify({ kind: 'receipt', hash: envelope.data.hash, + deliveries: envelope.data.served.length }, null, 2)); +} diff --git a/lib/commands/export.js b/lib/commands/export.js new file mode 100644 index 0000000..a0dba06 --- /dev/null +++ b/lib/commands/export.js @@ -0,0 +1,28 @@ +'use strict'; +const fs = require('fs'); +const store = require('../store'); +const { contextExport, encode, decode } = require('../export-format'); + +function exportCommand(args) { + if (args[0] === 'receipt' && args.length === 2) { + const { loadSessions } = require('./session'); + const matches = loadSessions(store.requireRoot()).filter(receipt => + receipt.id === args[1] || receipt.hash?.startsWith(args[1])); + if (!args[1] || matches.length !== 1) throw new Error('Receipt not found or selector is ambiguous.'); + process.stdout.write(encode({ format: 'agenctx-export', version: 1, + kind: 'receipt', data: matches[0] })); + return; + } + if (args[0] === 'validate' && args.length === 2) { + const envelope = decode(fs.readFileSync(args[1], 'utf8')); + console.log(JSON.stringify({ valid: true, version: envelope.version, kind: envelope.kind })); + return; + } + if (args.length !== 1 || args[0] !== 'context') { + throw new Error('Usage: agenctx export context | receipt | validate '); + } + const root = store.requireRoot(); + process.stdout.write(encode(contextExport(store.loadConfig(root), store.loadState(root)))); +} + +module.exports = { exportCommand }; diff --git a/lib/commands/search.js b/lib/commands/search.js index ffe1f97..32eef13 100644 --- a/lib/commands/search.js +++ b/lib/commands/search.js @@ -301,4 +301,4 @@ async function search(args) { }, { agent }); } -module.exports = { search }; +module.exports = { search, collectEntries, filterEntries }; diff --git a/lib/export-format.js b/lib/export-format.js new file mode 100644 index 0000000..8672254 --- /dev/null +++ b/lib/export-format.js @@ -0,0 +1,65 @@ +'use strict'; + +function object(value, name) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${name} must be an object.`); + } +} + +function validate(envelope) { + object(envelope, 'Export'); + if (envelope.format !== 'agenctx-export' || envelope.version !== 1) { + throw new Error('Unsupported agenctx export format/version.'); + } + object(envelope.data, 'Export data'); + if (envelope.kind === 'receipt') { + const receipt = envelope.data; + if (receipt.version !== 1 || typeof receipt.id !== 'string' || !receipt.id || + !Array.isArray(receipt.served) || typeof receipt.ended !== 'string' || + !Number.isFinite(Date.parse(receipt.ended))) { + throw new Error('Expected a sealed version-1 receipt.'); + } + if (!/^[0-9a-f]{64}$/.test(receipt.hash || '') || + require('./commands/session').receiptHash(receipt) !== receipt.hash) { + throw new Error('Receipt hash verification failed.'); + } + return envelope; + } + if (envelope.kind !== 'context') throw new Error('Unsupported export kind.'); + const { config, state } = envelope.data; + object(config, 'config'); + object(state, 'state'); + object(state.banners, 'state.banners'); + const ids = new Set(); + for (const [name, banner] of Object.entries(state.banners)) { + object(banner, `banner ${name}`); + if (!Array.isArray(banner.entries)) throw new Error(`banner ${name} needs entries.`); + for (const entry of banner.entries) { + object(entry, 'entry'); + if (typeof entry.id !== 'string' || !entry.id || ids.has(entry.id)) { + throw new Error('Entry IDs must be nonempty and globally unique.'); + } + ids.add(entry.id); + if (typeof entry.content !== 'string') throw new Error(`Entry ${entry.id} needs string content.`); + if (!['active', 'pinned', 'ambient', 'archived'].includes(entry.status)) { + throw new Error(`Entry ${entry.id} has an unsupported status.`); + } + } + } + return envelope; +} + +function encode(envelope) { + return JSON.stringify(validate(envelope), null, 2) + '\n'; +} + +function decode(text) { + return validate(JSON.parse(text.replace(/^\uFEFF/, ''))); +} + +function contextExport(config, state) { + return decode(JSON.stringify({ format: 'agenctx-export', version: 1, + kind: 'context', data: { config, state } })); +} + +module.exports = { validate, encode, decode, contextExport }; diff --git a/scripts/evaluate-retrieval.js b/scripts/evaluate-retrieval.js new file mode 100644 index 0000000..3060252 --- /dev/null +++ b/scripts/evaluate-retrieval.js @@ -0,0 +1,43 @@ +'use strict'; +const fs = require('fs'); +const path = require('path'); +const { collectEntries, filterEntries } = require('../lib/commands/search'); + +function evaluate(fixture) { + if (fixture.version !== 1 || !Array.isArray(fixture.cases) || !fixture.cases.length) { + throw new Error('Expected version 1 with nonempty retrieval cases.'); + } + const cases = fixture.cases.map(example => { + const actual = filterEntries(collectEntries(fixture.state, example), { + ...example, since: example.since ? new Date(example.since) : null, + }).map(({ entry }) => entry.id); + const expected = new Set(example.expected); + const hits = actual.filter(id => expected.has(id)).length; + return { + name: example.name, expected: [...expected], actual, + precision: actual.length ? hits / actual.length : (expected.size ? 0 : 1), + recall: expected.size ? hits / expected.size : 1, + exact: JSON.stringify(actual) === JSON.stringify(example.expected), + }; + }); + return { + version: 1, evidence: 'synthetic retrieval mechanics; not agent task performance', + cases, exact_match_rate: cases.filter(c => c.exact).length / cases.length, + macro_precision: cases.reduce((sum, c) => sum + c.precision, 0) / cases.length, + macro_recall: cases.reduce((sum, c) => sum + c.recall, 0) / cases.length, + }; +} + +if (require.main === module) { + try { + const file = process.argv[2] || path.join(__dirname, '../test/fixtures/retrieval.json'); + const report = evaluate(JSON.parse(fs.readFileSync(file, 'utf8'))); + console.log(JSON.stringify(report, null, 2)); + if (report.exact_match_rate !== 1) process.exitCode = 1; + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} + +module.exports = { evaluate }; diff --git a/test/export-cli.test.js b/test/export-cli.test.js new file mode 100644 index 0000000..444d5dd --- /dev/null +++ b/test/export-cli.test.js @@ -0,0 +1,30 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const cli = path.resolve(__dirname, '../bin/agenctx.js'); + +test('CLI exports pure JSON without modifying store and validates outside a project', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agenctx-export-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const run = (...args) => spawnSync(process.execPath, [cli, ...args], { cwd: root, encoding: 'utf8' }); + fs.mkdirSync(path.join(root, '.agenctx')); + const statePath = path.join(root, '.agenctx/state.json'); + fs.writeFileSync(statePath, JSON.stringify(require('./fixtures/retrieval.json').state)); + fs.writeFileSync(path.join(root, '.agenctx/config.json'), '{}'); + const before = fs.readFileSync(statePath, 'utf8'); + const result = run('export', 'context'); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).kind, 'context'); + assert.equal(fs.readFileSync(statePath, 'utf8'), before); + const exported = path.join(root, 'export.json'); + fs.writeFileSync(exported, result.stdout); + const validated = spawnSync(process.execPath, [cli, 'export', 'validate', exported], + { cwd: os.tmpdir(), encoding: 'utf8' }); + assert.equal(validated.status, 0, validated.stderr); + assert.equal(JSON.parse(validated.stdout).valid, true); + assert.notEqual(run('export', 'nonsense').status, 0); +}); diff --git a/test/export-format.test.js b/test/export-format.test.js new file mode 100644 index 0000000..9090d8e --- /dev/null +++ b/test/export-format.test.js @@ -0,0 +1,44 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { contextExport, encode, decode } = require('../lib/export-format'); +const fixture = require('./fixtures/retrieval.json'); +const { receiptHash } = require('../lib/commands/session'); + +test('receipt exports verify original content hash and retain event order', () => { + const data = { version: 1, id: 's-test', parent: null, actor: 'agent', description: 'test', + started: '2026-01-01T00:00:00Z', ended: '2026-01-01T00:01:00Z', duration: 60, + served: [{ sequence: 1, entries: [] }, { sequence: 2, entries: [] }] }; + data.hash = receiptHash(data); + const envelope = { format: 'agenctx-export', version: 1, kind: 'receipt', data }; + assert.deepEqual(decode(encode(envelope)), envelope); + data.served.reverse(); + assert.throws(() => encode(envelope), /hash verification/); + delete data.hash; + assert.throws(() => encode(envelope), /hash verification/); +}); + +test('context export round trip preserves statuses and Unicode without aliasing', () => { + const state = JSON.parse(JSON.stringify(fixture.state)); + state.banners.rules.entries[0].content += '\n日本語 🦙'; + const exported = contextExport({ version: 1 }, state); + assert.deepEqual(decode(encode(exported)), exported); + exported.data.state.banners.rules.entries[0].content = 'changed'; + assert.notEqual(state.banners.rules.entries[0].content, 'changed'); +}); + +test('reject unknown versions, malformed entries, duplicate IDs and kinds', () => { + const valid = contextExport({}, fixture.state); + for (const modify of [ + e => { e.version = 2; }, + e => { e.kind = 'unknown'; }, + e => { e.data.state.banners.rules.entries[0].content = 42; }, + e => { e.data.state.banners.rules.entries[0].status = 'trusted'; }, + e => { e.data.state.banners.testing.entries[0].id = 'aa0001'; }, + ]) { + const changed = JSON.parse(encode(valid)); + modify(changed); + assert.throws(() => decode(JSON.stringify(changed))); + } + assert.throws(() => decode('{bad json')); +}); diff --git a/test/fixtures/retrieval.json b/test/fixtures/retrieval.json new file mode 100644 index 0000000..4070bcd --- /dev/null +++ b/test/fixtures/retrieval.json @@ -0,0 +1,26 @@ +{ + "version": 1, + "state": { + "banners": { + "rules": {"entries": [ + {"id": "aa0001", "content": "Authentication uses JWT.\nValidate audience and issuer.", "status": "pinned", "created": "2026-01-01T00:00:00Z"}, + {"id": "aa0002", "content": "Use parameterized SQL queries.", "status": "active", "created": "2026-02-01T00:00:00Z"}, + {"id": "aa0003", "content": "JWT legacy migration notes", "status": "archived", "created": "2026-01-01T00:00:00Z"} + ]}, + "testing": {"entries": [ + {"id": "bb0001", "content": "Run JWT integration tests before release.", "status": "active", "created": "2026-03-01T00:00:00Z"}, + {"id": "bb0002", "content": "Historical SQL benchmark", "status": "ambient", "created": "2026-01-01T00:00:00Z"} + ]} + } + }, + "cases": [ + {"name": "keyword across banners", "query": "JWT", "expected": ["aa0001", "bb0001"]}, + {"name": "case insensitive", "query": "jwt", "expected": ["aa0001", "bb0001"]}, + {"name": "banner scope", "query": "JWT", "banner": "rules", "expected": ["aa0001"]}, + {"name": "date scope", "query": "JWT", "since": "2026-02-01T00:00:00Z", "expected": ["bb0001"]}, + {"name": "hidden ambient", "query": "SQL", "expected": ["aa0002"]}, + {"name": "explicit ambient", "query": "SQL", "includeAmbient": true, "expected": ["aa0002", "bb0002"]}, + {"name": "no match", "query": "kubernetes", "expected": []}, + {"name": "full multiline match", "query": "audience", "expected": ["aa0001"]} + ] +} diff --git a/test/retrieval-report.test.js b/test/retrieval-report.test.js new file mode 100644 index 0000000..0e1061a --- /dev/null +++ b/test/retrieval-report.test.js @@ -0,0 +1,16 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fixture = require('./fixtures/retrieval.json'); +const { evaluate } = require('../scripts/evaluate-retrieval'); + +test('retrieval report is deterministic and counts missed expectations', () => { + assert.deepEqual(evaluate(fixture), evaluate(fixture)); + assert.equal(evaluate(fixture).exact_match_rate, 1); + const changed = JSON.parse(JSON.stringify(fixture)); + changed.cases[0].expected.push('missing'); + const report = evaluate(changed); + assert.ok(report.exact_match_rate < 1); + assert.equal(report.cases[0].recall, 2 / 3); + assert.throws(() => evaluate({ version: 2, cases: [] }), /version 1/); +}); diff --git a/test/retrieval.test.js b/test/retrieval.test.js new file mode 100644 index 0000000..f34f9b8 --- /dev/null +++ b/test/retrieval.test.js @@ -0,0 +1,23 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fixture = require('./fixtures/retrieval.json'); +const { collectEntries, filterEntries } = require('../lib/commands/search'); +const { findEntry } = require('../lib/store'); + +for (const example of fixture.cases) { + test(`retrieval: ${example.name}`, () => { + const before = JSON.stringify(fixture.state); + const entries = collectEntries(fixture.state, example); + const actual = filterEntries(entries, { + ...example, since: example.since ? new Date(example.since) : null, + }).map(({ entry }) => entry.id); + assert.deepEqual(actual, example.expected); + assert.equal(JSON.stringify(fixture.state), before); + }); +} +test('exact ID lookup preserves full multiline content', () => { + assert.equal(findEntry(fixture.state, 'aa0001').entry.content, + 'Authentication uses JWT.\nValidate audience and issuer.'); + assert.equal(findEntry(fixture.state, 'ffffff'), null); +});