feat(storage): reclaim free runtime SQLite pages in Host maintenance

packages/runtime-host/src/__tests__/storage-maintenance.test.ts

@@ -264,3 +264,42 @@ test('context-offload page reclamation lane runs with bounded batches', async (t
   assert.equal(reclaimCalls, 2);
   await maintenance.close();
 });
+
+test('runtime page reclamation lane starts after Ready and idles when drained', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  let calls = 0;
+  const maintenance = new HostStorageMaintenance({
+    artifacts: {
+      reclaimUpgradeResidue: async () => ({
+        nextAfter: null,
+        processedPaths: 0,
+        failedPaths: 0,
+      }),
+    },
+    runtimePageReclamation: {
+      reclaimFreePages: async ({ maxPages }) => {
+        assert.equal(maxPages, 64);
+        calls += 1;
+        return { reclaimedPages: calls === 1 ? 64 : 0, hasMore: calls === 1 };
+      },
+    },
+    onError: assert.fail,
+  });
+  t.mock.timers.tick(100);
+  await settle();
+  assert.equal(calls, 0);
+  maintenance.start();
+  t.mock.timers.tick(100);
+  await settle();
+  assert.equal(calls, 1);
+  t.mock.timers.tick(100);
+  await settle();
+  assert.equal(calls, 2);
+  t.mock.timers.tick(59_999);
+  await settle();
+  assert.equal(calls, 2);
+  t.mock.timers.tick(1);
+  await settle();
+  assert.equal(calls, 3);
+  await maintenance.close();
+});

packages/runtime-host/src/server/execution-composition.ts

@@ -2905,6 +2905,7 @@ export async function createExecutionRuntimeHostComposition(
     const storageMaintenance = new HostStorageMaintenance({
       artifacts: openedArtifactStore,
       contextOffload: openedContextOffloadStore,
+      runtimePageReclamation: storage.runtimePageReclamation,
       retention: archiveRetention,
       onError: (name, error) =>
         console.error(`[runtime-host] ${name} will retry: ${generalizedErrorMessage(error)}`),

packages/runtime-host/src/server/storage-maintenance.ts

@@ -51,6 +51,12 @@ export class HostStorageMaintenance {
   constructor(input: {
     artifacts: Pick<InteractiveArtifactStoreWriter, 'reclaimUpgradeResidue'>;
     contextOffload?: Pick<InteractiveContextOffloadWriter, 'collectGarbage' | 'reclaimFreePages'>;
+    runtimePageReclamation?: {
+      reclaimFreePages(input: { readonly maxPages: number }): Promise<{
+        readonly reclaimedPages: number;
+        readonly hasMore: boolean;
+      }>;
+    };
     /** The opt-in archived-task retention sweep; true while candidates remain. */
     retention?: { sweep(): Promise<boolean> };
     onError: (name: string, error: unknown) => void;
@@ -106,6 +112,16 @@ export class HostStorageMaintenance {
         },
       });
     }
+    const runtime = input.runtimePageReclamation;
+    if (runtime) {
+      this.#lanes.push({
+        name: 'runtime page reclamation',
+        failures: 0,
+        activeDelay: ACTIVE_DELAY_MS,
+        idleDelay: IDLE_DELAY_MS,
+        run: async () => (await runtime.reclaimFreePages({ maxPages: MAX_VACUUM_PAGES })).hasMore,
+      });
+    }
     const retention = input.retention;
     if (retention)
       this.#lanes.push({

packages/storage/src/__tests__/runtime-page-reclamation.test.ts

@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import assert from 'node:assert/strict';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { after, test } from 'node:test';
+import { acquireOperationalStateDatabase } from '../operational-state-store.js';
+import { openRuntimePageReclamation } from '../runtime-page-reclamation.js';
+import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '../root-authority.js';
+import {
+  removeTrackedControlDirectories,
+  trackControlDirectory,
+} from './fixtures/control-directory-hygiene.js';
+
+after(removeTrackedControlDirectories);
+
+async function withRuntimeDatabase(
+  existingNone: boolean,
+  run: (input: {
+    root: string;
+    database: DatabaseSync;
+    reclaim: ReturnType<typeof openRuntimePageReclamation>;
+  }) => Promise<void>,
+): Promise<void> {
+  const root = await mkdtemp(join(tmpdir(), 'maka-runtime-reclamation-'));
+  if (existingNone) {
+    acquireOperationalStateDatabase(root).close();
+    const existing = new DatabaseSync(join(root, 'runtime.sqlite'));
+    try {
+      existing.exec('PRAGMA auto_vacuum = NONE; VACUUM');
+    } finally {
+      existing.close();
+    }
+  }
+  try {
+    const capability = trackControlDirectory(
+      await resolveStorageRoot({ path: root, kind: 'interactive' }),
+    );
+    const owner = await tryAcquireInteractiveRootOwner(capability);
+    assert.ok(owner);
+    try {
+      const lease = acquireOperationalStateDatabase(root);
+      try {
+        await run({
+          root,
+          database: lease.database,
+          reclaim: openRuntimePageReclamation(owner.lease),
+        });
+      } finally {
+        lease.close();
+      }
+    } finally {
+      await owner.close();
+    }
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+function pragma(database: DatabaseSync, name: 'auto_vacuum' | 'freelist_count'): number {
+  return (database.prepare(`PRAGMA ${name}`).get() as Record<typeof name, number>)[name];
+}
+
+function createFreePages(database: DatabaseSync): number {
+  database.exec('CREATE TABLE reclaim_fixture (payload BLOB NOT NULL)');
+  const insert = database.prepare('INSERT INTO reclaim_fixture VALUES (?)');
+  database.exec('BEGIN');
+  try {
+    for (let index = 0; index < 120; index += 1) insert.run(Buffer.alloc(8192, index));
+    database.exec('COMMIT');
+  } catch (error) {
+    database.exec('ROLLBACK');
+    throw error;
+  }
+  database.exec('DELETE FROM reclaim_fixture');
+  return pragma(database, 'freelist_count');
+}
+
+test('runtime reclamation frees at most the page budget per tick and drains incremental databases', async () => {
+  await withRuntimeDatabase(false, async ({ database, reclaim }) => {
+    assert.equal(pragma(database, 'auto_vacuum'), 2);
+    const initial = createFreePages(database);
+    assert.ok(initial > 64);
+
+    const first = await reclaim.reclaimFreePages({ maxPages: 64 });
+    assert.ok(first.reclaimedPages > 0 && first.reclaimedPages <= 64);
+    assert.equal(first.hasMore, true);
+    assert.equal(pragma(database, 'freelist_count'), initial - first.reclaimedPages);
+
+    let remaining = pragma(database, 'freelist_count');
+    for (let tick = 0; tick < 100 && remaining > 0; tick += 1) {
+      const result = await reclaim.reclaimFreePages({ maxPages: 64 });
+      assert.ok(result.reclaimedPages > 0 && result.reclaimedPages <= 64);
+      remaining = pragma(database, 'freelist_count');
+      assert.equal(result.hasMore, remaining > 0);
+    }
+    assert.equal(remaining, 0);
+    assert.deepEqual(await reclaim.reclaimFreePages({ maxPages: 64 }), {
+      reclaimedPages: 0,
+      hasMore: false,
+    });
+  });
+});
+
+test('runtime reclamation leaves an existing NONE database and its freelist untouched', async () => {
+  await withRuntimeDatabase(true, async ({ database, reclaim }) => {
+    assert.equal(pragma(database, 'auto_vacuum'), 0);
+    const before = createFreePages(database);
+    assert.ok(before > 0);
+    assert.deepEqual(await reclaim.reclaimFreePages({ maxPages: 64 }), {
+      reclaimedPages: 0,
+      hasMore: false,
+    });
+    assert.equal(pragma(database, 'auto_vacuum'), 0);
+    assert.equal(pragma(database, 'freelist_count'), before);
+  });
+});
+
+test('runtime reclamation rejects unbounded page counts', async () => {
+  await withRuntimeDatabase(false, async ({ reclaim }) => {
+    assert.throws(() => reclaim.reclaimFreePages({ maxPages: 0 }), /positive safe integer/);
+    assert.throws(
+      () => reclaim.reclaimFreePages({ maxPages: Number.MAX_SAFE_INTEGER + 1 }),
+      /positive safe integer/,
+    );
+  });
+});

packages/storage/src/runtime-page-reclamation.ts

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  acquireOperationalStateDatabase,
+  type OperationalStateDatabaseLease,
+} from './operational-state-store.js';
+import { runWithStorageRootLease, type StorageRootLease } from './root-authority.js';
+
+export interface RuntimePageReclamation {
+  reclaimFreePages(input: { readonly maxPages: number }): Promise<{
+    readonly reclaimedPages: number;
+    readonly hasMore: boolean;
+  }>;
+}
+
+/** Uses the process-local operational owner; older NONE databases remain report-only. */
+export function openRuntimePageReclamation(
+  rootLease: StorageRootLease<'interactive', 'write'>,
+): RuntimePageReclamation {
+  return Object.freeze({
+    reclaimFreePages: ({ maxPages }: { readonly maxPages: number }) => {
+      if (!Number.isSafeInteger(maxPages) || maxPages <= 0) {
+        throw new Error('Runtime page reclamation limit must be a positive safe integer');
+      }
+      return runWithStorageRootLease(rootLease, 'interactive', 'write', async (root) => {
+        const databaseLease = acquireOperationalStateDatabase(root, {
+          schemaMigration: 'require_current',
+        });
+        try {
+          return reclaimFreePages(databaseLease, maxPages);
+        } finally {
+          databaseLease.close();
+        }
+      });
+    },
+  });
+}
+
+function reclaimFreePages(
+  lease: OperationalStateDatabaseLease,
+  maxPages: number,
+): { readonly reclaimedPages: number; readonly hasMore: boolean } {
+  // An idle tick and an older NONE database must never acquire a write lock.
+  const eligible = lease.transaction('read', () => {
+    const mode = readPragma(lease, 'auto_vacuum');
+    if (mode !== 2) return false;
+    return readPragma(lease, 'freelist_count') > 0;
+  });
+  if (!eligible) return { reclaimedPages: 0, hasMore: false };
+
+  const result = lease.transaction('write', () => {
+    const before = readPragma(lease, 'freelist_count');
+    // incremental_vacuum(0) would reclaim every page, so stop if the list emptied.
+    if (before === 0) return { reclaimedPages: 0, hasMore: false };
+    lease.database.exec(`PRAGMA incremental_vacuum(${Math.min(maxPages, before)})`);
+    const after = readPragma(lease, 'freelist_count');
+    return { reclaimedPages: before - after, hasMore: after > 0 && after < before };
+  });
+  // PASSIVE does not wait for other readers; it runs after the write commits.
+  if (result.reclaimedPages > 0) lease.database.exec('PRAGMA wal_checkpoint(PASSIVE)');
+  return result;
+}
+
+function readPragma(lease: OperationalStateDatabaseLease, name: 'auto_vacuum' | 'freelist_count') {
+  const row = lease.database.prepare(`PRAGMA ${name}`).get() as Record<string, unknown> | undefined;
+  const value = row?.[name];
+  if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`Invalid SQLite ${name}`);
+  }
+  return value;
+}

packages/storage/src/storage-footprint.ts

@@ -50,7 +50,7 @@ export interface StorageFootprintTotal {
 
 export interface StorageFootprint {
   readonly totals: readonly StorageFootprintTotal[];
-  /** Free pages in runtime.sqlite that a future VACUUM could return to the OS. */
+  /** Free pages in runtime.sqlite; incremental databases can reclaim them in maintenance. */
   readonly reclaimableBytes: number;
   /** Live Subagent worktrees; counted, never walked, because a checkout can be huge. */
   readonly worktreeCount: number;

packages/storage/src/storage-writer-composition.ts

@@ -38,6 +38,10 @@ import {
   type StorageRootLease,
 } from './root-authority.js';
 import { openInteractiveRuntimePolicyStoresForWrite } from './runtime-policy-stores.js';
+import {
+  openRuntimePageReclamation,
+  type RuntimePageReclamation,
+} from './runtime-page-reclamation.js';
 import { openInteractiveScheduledTaskStoreForWrite } from './scheduled-task-store.js';
 import { openInteractiveSessionTodoStoreForWrite } from './session-todo-authority.js';
 import { openInteractiveShellRunStoreForWrite } from './shell-run-authority.js';
@@ -84,6 +88,7 @@ export interface StorageWriterComposition {
   readonly usage: Awaited<ReturnType<typeof openInteractiveUsageStoresForWrite>>;
   readonly shellRuns: Awaited<ReturnType<typeof openInteractiveShellRunStoreForWrite>>;
   readonly footprint: InteractiveStorageFootprintReader;
+  readonly runtimePageReclamation: RuntimePageReclamation;
   /** The archived-task retention document, bound to the composition's write lease. */
   readonly archiveRetention: InteractiveArchiveRetentionStore;
   close(): Promise<void>;
@@ -239,6 +244,7 @@ async function createComposition(
     usage,
     shellRuns,
     footprint,
+    runtimePageReclamation: openRuntimePageReclamation(lease),
     archiveRetention: openArchiveRetentionStore(lease),
     close,
   });