From 5171173f10b3bdef9180e30146d50449a882ed74 Mon Sep 17 00:00:00 2001 From: Garvit Arora Date: Mon, 12 Oct 2026 08:30:30 +0530 Subject: [PATCH 1/2] feat(storage): reclaim runtime SQLite free pages in bounded Host lane Generated-by: Codex (GPT-6) --- .../src/server/execution-composition.ts | 1 + .../src/server/storage-maintenance.ts | 16 ++++ .../storage/src/runtime-page-reclamation.ts | 88 +++++++++++++++++++ packages/storage/src/storage-footprint.ts | 2 +- .../storage/src/storage-writer-composition.ts | 6 ++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 packages/storage/src/runtime-page-reclamation.ts diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 2771c1605c6..94021b0be00 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -2905,6 +2905,7 @@ export async function createExecutionRuntimeHostComposition( const storageMaintenance = new HostStorageMaintenance({ artifacts: openedArtifactStore, contextOffload: openedContextOffloadStore, + runtimePageReclamation: storage.runtimePageReclamation, retention: archiveRetention, onError: (name, error) => console.error(`[runtime-host] ${name} will retry: ${generalizedErrorMessage(error)}`), diff --git a/packages/runtime-host/src/server/storage-maintenance.ts b/packages/runtime-host/src/server/storage-maintenance.ts index 5770fb46deb..623e91d082d 100644 --- a/packages/runtime-host/src/server/storage-maintenance.ts +++ b/packages/runtime-host/src/server/storage-maintenance.ts @@ -51,6 +51,12 @@ export class HostStorageMaintenance { constructor(input: { artifacts: Pick; contextOffload?: Pick; + runtimePageReclamation?: { + reclaimFreePages(input: { readonly maxPages: number }): Promise<{ + readonly reclaimedPages: number; + readonly hasMore: boolean; + }>; + }; /** The opt-in archived-task retention sweep; true while candidates remain. */ retention?: { sweep(): Promise }; onError: (name: string, error: unknown) => void; @@ -106,6 +112,16 @@ export class HostStorageMaintenance { }, }); } + const runtime = input.runtimePageReclamation; + if (runtime) { + this.#lanes.push({ + name: 'runtime page reclamation', + failures: 0, + activeDelay: ACTIVE_DELAY_MS, + idleDelay: IDLE_DELAY_MS, + run: async () => (await runtime.reclaimFreePages({ maxPages: MAX_VACUUM_PAGES })).hasMore, + }); + } const retention = input.retention; if (retention) this.#lanes.push({ diff --git a/packages/storage/src/runtime-page-reclamation.ts b/packages/storage/src/runtime-page-reclamation.ts new file mode 100644 index 00000000000..24380f48cbd --- /dev/null +++ b/packages/storage/src/runtime-page-reclamation.ts @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + acquireOperationalStateDatabase, + type OperationalStateDatabaseLease, +} from './operational-state-store.js'; +import { runWithStorageRootLease, type StorageRootLease } from './root-authority.js'; + +export interface RuntimePageReclamation { + reclaimFreePages(input: { readonly maxPages: number }): Promise<{ + readonly reclaimedPages: number; + readonly hasMore: boolean; + }>; +} + +/** Uses the process-local operational owner; older NONE databases remain report-only. */ +export function openRuntimePageReclamation( + rootLease: StorageRootLease<'interactive', 'write'>, +): RuntimePageReclamation { + return Object.freeze({ + reclaimFreePages: ({ maxPages }: { readonly maxPages: number }) => { + if (!Number.isSafeInteger(maxPages) || maxPages <= 0) { + throw new Error('Runtime page reclamation limit must be a positive safe integer'); + } + return runWithStorageRootLease(rootLease, 'interactive', 'write', async (root) => { + const databaseLease = acquireOperationalStateDatabase(root, { + schemaMigration: 'require_current', + }); + try { + return reclaimFreePages(databaseLease, maxPages); + } finally { + databaseLease.close(); + } + }); + }, + }); +} + +function reclaimFreePages( + lease: OperationalStateDatabaseLease, + maxPages: number, +): { readonly reclaimedPages: number; readonly hasMore: boolean } { + // An idle tick and an older NONE database must never acquire a write lock. + const eligible = lease.transaction('read', () => { + const mode = readPragma(lease, 'auto_vacuum'); + if (mode !== 2) return false; + return readPragma(lease, 'freelist_count') > 0; + }); + if (!eligible) return { reclaimedPages: 0, hasMore: false }; + + const result = lease.transaction('write', () => { + const before = readPragma(lease, 'freelist_count'); + // incremental_vacuum(0) would reclaim every page, so stop if the list emptied. + if (before === 0) return { reclaimedPages: 0, hasMore: false }; + lease.database.exec(`PRAGMA incremental_vacuum(${Math.min(maxPages, before)})`); + const after = readPragma(lease, 'freelist_count'); + return { reclaimedPages: before - after, hasMore: after > 0 && after < before }; + }); + // PASSIVE does not wait for other readers; it runs after the write commits. + if (result.reclaimedPages > 0) lease.database.exec('PRAGMA wal_checkpoint(PASSIVE)'); + return result; +} + +function readPragma(lease: OperationalStateDatabaseLease, name: 'auto_vacuum' | 'freelist_count') { + const row = lease.database.prepare(`PRAGMA ${name}`).get() as Record | undefined; + const value = row?.[name]; + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid SQLite ${name}`); + } + return value; +} diff --git a/packages/storage/src/storage-footprint.ts b/packages/storage/src/storage-footprint.ts index 86f94f9b929..6afad29664a 100644 --- a/packages/storage/src/storage-footprint.ts +++ b/packages/storage/src/storage-footprint.ts @@ -50,7 +50,7 @@ export interface StorageFootprintTotal { export interface StorageFootprint { readonly totals: readonly StorageFootprintTotal[]; - /** Free pages in runtime.sqlite that a future VACUUM could return to the OS. */ + /** Free pages in runtime.sqlite; incremental databases can reclaim them in maintenance. */ readonly reclaimableBytes: number; /** Live Subagent worktrees; counted, never walked, because a checkout can be huge. */ readonly worktreeCount: number; diff --git a/packages/storage/src/storage-writer-composition.ts b/packages/storage/src/storage-writer-composition.ts index 85d4e318320..814ddfb6900 100644 --- a/packages/storage/src/storage-writer-composition.ts +++ b/packages/storage/src/storage-writer-composition.ts @@ -38,6 +38,10 @@ import { type StorageRootLease, } from './root-authority.js'; import { openInteractiveRuntimePolicyStoresForWrite } from './runtime-policy-stores.js'; +import { + openRuntimePageReclamation, + type RuntimePageReclamation, +} from './runtime-page-reclamation.js'; import { openInteractiveScheduledTaskStoreForWrite } from './scheduled-task-store.js'; import { openInteractiveSessionTodoStoreForWrite } from './session-todo-authority.js'; import { openInteractiveShellRunStoreForWrite } from './shell-run-authority.js'; @@ -84,6 +88,7 @@ export interface StorageWriterComposition { readonly usage: Awaited>; readonly shellRuns: Awaited>; readonly footprint: InteractiveStorageFootprintReader; + readonly runtimePageReclamation: RuntimePageReclamation; /** The archived-task retention document, bound to the composition's write lease. */ readonly archiveRetention: InteractiveArchiveRetentionStore; close(): Promise; @@ -239,6 +244,7 @@ async function createComposition( usage, shellRuns, footprint, + runtimePageReclamation: openRuntimePageReclamation(lease), archiveRetention: openArchiveRetentionStore(lease), close, }); From 893338cdbeaed6e5a29d852f03038f7d985c9ca6 Mon Sep 17 00:00:00 2001 From: Garvit Arora Date: Tue, 13 Oct 2026 08:17:27 +0530 Subject: [PATCH 2/2] test(storage): cover bounded runtime page reclamation Generated-by: Codex (GPT-6) --- .../src/__tests__/storage-maintenance.test.ts | 39 +++++ .../runtime-page-reclamation.test.ts | 146 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 packages/storage/src/__tests__/runtime-page-reclamation.test.ts diff --git a/packages/runtime-host/src/__tests__/storage-maintenance.test.ts b/packages/runtime-host/src/__tests__/storage-maintenance.test.ts index fd6f240fbc4..fbcedb2d971 100644 --- a/packages/runtime-host/src/__tests__/storage-maintenance.test.ts +++ b/packages/runtime-host/src/__tests__/storage-maintenance.test.ts @@ -264,3 +264,42 @@ test('context-offload page reclamation lane runs with bounded batches', async (t assert.equal(reclaimCalls, 2); await maintenance.close(); }); + +test('runtime page reclamation lane starts after Ready and idles when drained', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let calls = 0; + const maintenance = new HostStorageMaintenance({ + artifacts: { + reclaimUpgradeResidue: async () => ({ + nextAfter: null, + processedPaths: 0, + failedPaths: 0, + }), + }, + runtimePageReclamation: { + reclaimFreePages: async ({ maxPages }) => { + assert.equal(maxPages, 64); + calls += 1; + return { reclaimedPages: calls === 1 ? 64 : 0, hasMore: calls === 1 }; + }, + }, + onError: assert.fail, + }); + t.mock.timers.tick(100); + await settle(); + assert.equal(calls, 0); + maintenance.start(); + t.mock.timers.tick(100); + await settle(); + assert.equal(calls, 1); + t.mock.timers.tick(100); + await settle(); + assert.equal(calls, 2); + t.mock.timers.tick(59_999); + await settle(); + assert.equal(calls, 2); + t.mock.timers.tick(1); + await settle(); + assert.equal(calls, 3); + await maintenance.close(); +}); diff --git a/packages/storage/src/__tests__/runtime-page-reclamation.test.ts b/packages/storage/src/__tests__/runtime-page-reclamation.test.ts new file mode 100644 index 00000000000..221dae8ad0f --- /dev/null +++ b/packages/storage/src/__tests__/runtime-page-reclamation.test.ts @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { after, test } from 'node:test'; +import { acquireOperationalStateDatabase } from '../operational-state-store.js'; +import { openRuntimePageReclamation } from '../runtime-page-reclamation.js'; +import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '../root-authority.js'; +import { + removeTrackedControlDirectories, + trackControlDirectory, +} from './fixtures/control-directory-hygiene.js'; + +after(removeTrackedControlDirectories); + +async function withRuntimeDatabase( + existingNone: boolean, + run: (input: { + root: string; + database: DatabaseSync; + reclaim: ReturnType; + }) => Promise, +): Promise { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-reclamation-')); + if (existingNone) { + acquireOperationalStateDatabase(root).close(); + const existing = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + existing.exec('PRAGMA auto_vacuum = NONE; VACUUM'); + } finally { + existing.close(); + } + } + try { + const capability = trackControlDirectory( + await resolveStorageRoot({ path: root, kind: 'interactive' }), + ); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + try { + const lease = acquireOperationalStateDatabase(root); + try { + await run({ + root, + database: lease.database, + reclaim: openRuntimePageReclamation(owner.lease), + }); + } finally { + lease.close(); + } + } finally { + await owner.close(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +function pragma(database: DatabaseSync, name: 'auto_vacuum' | 'freelist_count'): number { + return (database.prepare(`PRAGMA ${name}`).get() as Record)[name]; +} + +function createFreePages(database: DatabaseSync): number { + database.exec('CREATE TABLE reclaim_fixture (payload BLOB NOT NULL)'); + const insert = database.prepare('INSERT INTO reclaim_fixture VALUES (?)'); + database.exec('BEGIN'); + try { + for (let index = 0; index < 120; index += 1) insert.run(Buffer.alloc(8192, index)); + database.exec('COMMIT'); + } catch (error) { + database.exec('ROLLBACK'); + throw error; + } + database.exec('DELETE FROM reclaim_fixture'); + return pragma(database, 'freelist_count'); +} + +test('runtime reclamation frees at most the page budget per tick and drains incremental databases', async () => { + await withRuntimeDatabase(false, async ({ database, reclaim }) => { + assert.equal(pragma(database, 'auto_vacuum'), 2); + const initial = createFreePages(database); + assert.ok(initial > 64); + + const first = await reclaim.reclaimFreePages({ maxPages: 64 }); + assert.ok(first.reclaimedPages > 0 && first.reclaimedPages <= 64); + assert.equal(first.hasMore, true); + assert.equal(pragma(database, 'freelist_count'), initial - first.reclaimedPages); + + let remaining = pragma(database, 'freelist_count'); + for (let tick = 0; tick < 100 && remaining > 0; tick += 1) { + const result = await reclaim.reclaimFreePages({ maxPages: 64 }); + assert.ok(result.reclaimedPages > 0 && result.reclaimedPages <= 64); + remaining = pragma(database, 'freelist_count'); + assert.equal(result.hasMore, remaining > 0); + } + assert.equal(remaining, 0); + assert.deepEqual(await reclaim.reclaimFreePages({ maxPages: 64 }), { + reclaimedPages: 0, + hasMore: false, + }); + }); +}); + +test('runtime reclamation leaves an existing NONE database and its freelist untouched', async () => { + await withRuntimeDatabase(true, async ({ database, reclaim }) => { + assert.equal(pragma(database, 'auto_vacuum'), 0); + const before = createFreePages(database); + assert.ok(before > 0); + assert.deepEqual(await reclaim.reclaimFreePages({ maxPages: 64 }), { + reclaimedPages: 0, + hasMore: false, + }); + assert.equal(pragma(database, 'auto_vacuum'), 0); + assert.equal(pragma(database, 'freelist_count'), before); + }); +}); + +test('runtime reclamation rejects unbounded page counts', async () => { + await withRuntimeDatabase(false, async ({ reclaim }) => { + assert.throws(() => reclaim.reclaimFreePages({ maxPages: 0 }), /positive safe integer/); + assert.throws( + () => reclaim.reclaimFreePages({ maxPages: Number.MAX_SAFE_INTEGER + 1 }), + /positive safe integer/, + ); + }); +});