From 8b62722f17e4f79173e6c76e906f0a85d92d2646 Mon Sep 17 00:00:00 2001 From: ApiliumDevTeam Date: Tue, 14 Jul 2026 18:59:58 +0200 Subject: [PATCH 1/3] feat(search): FTS5-backed session search with BM25 ranking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the linear substring scan with a dedicated SQLite FTS5 index for session message search. The index lives at ~/.mayros/agents//sessions/search.sqlite and uses an external-content virtual table with triggers for automatic sync. The search now supports multi-word queries with BM25 relevance ranking, role/timestamp/session filters, and snippet generation via the FTS5 snippet() function. Indexing is incremental (lazy sync on each query, skipping unchanged files by mtime+size), so no background watchers are needed. The public searchSessions API is unchanged — it tries FTS5 first and falls back to the original linear scan when node:sqlite is unavailable, so search always works regardless of the runtime. --- src/infra/session-search-fts.test.ts | 203 +++++++++++++++++++ src/infra/session-search-fts.ts | 279 +++++++++++++++++++++++++++ src/infra/session-search.ts | 43 +++++ 3 files changed, 525 insertions(+) create mode 100644 src/infra/session-search-fts.test.ts create mode 100644 src/infra/session-search-fts.ts diff --git a/src/infra/session-search-fts.test.ts b/src/infra/session-search-fts.test.ts new file mode 100644 index 00000000..db4ab80f --- /dev/null +++ b/src/infra/session-search-fts.test.ts @@ -0,0 +1,203 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { openSearchDb, syncSearchIndex, searchSessionsFTS } from "./session-search-fts.js"; + +/** + * FTS5 session search tests. + * + * Exercises the full path: schema creation, file indexing, and BM25-ranked + * search with filters. Uses temp directories and real SQLite databases. + */ + +let tempDir: string; +let dbPath: string; + +function setupTempSessions(files: Record): { sessionsDir: string; dbPath: string } { + tempDir = mkdtempSync(join(tmpdir(), "mayros-fts-test-")); + const sessionsDir = join(tempDir, "sessions"); + const { mkdirSync } = require("node:fs") as typeof import("node:fs"); + mkdirSync(sessionsDir, { recursive: true }); + for (const [name, content] of Object.entries(files)) { + writeFileSync(join(sessionsDir, name), content); + } + dbPath = join(sessionsDir, "search.sqlite"); + return { sessionsDir, dbPath }; +} + +afterEach(() => { + if (tempDir) { + try { + rmSync(tempDir, { recursive: true, force: true }); + } catch { + // best effort + } + } +}); + +function makeJsonLine( + type: string, + id: string, + role: string, + content: string, + timestamp: number, +): string { + return JSON.stringify({ + type, + id, + message: { role, content, timestamp }, + }); +} + +describe("FTS5 session search", () => { + it("creates the schema and indexes a session file", () => { + const { sessionsDir, dbPath } = setupTempSessions({ + "sess-1.jsonl": + [ + makeJsonLine("session", "hdr", "", "", 0), + makeJsonLine( + "message", + "m1", + "user", + "How do I configure authentication?", + 1700000000000, + ), + makeJsonLine( + "message", + "m2", + "assistant", + "You can use JWT tokens for auth.", + 1700000001000, + ), + ].join("\n") + "\n", + }); + + const db = openSearchDb(dbPath); + try { + const count = syncSearchIndex(db, sessionsDir, "main"); + expect(count).toBe(1); // one file indexed + + const rows = db.prepare("SELECT COUNT(*) as n FROM session_messages").get() as { n: number }; + expect(rows.n).toBe(2); // user + assistant messages (session header skipped) + } finally { + db.close(); + } + }); + + it("returns BM25-ranked results for a keyword query", () => { + const { sessionsDir, dbPath } = setupTempSessions({ + "sess-1.jsonl": + [ + makeJsonLine( + "message", + "m1", + "user", + "How do I configure authentication?", + 1700000000000, + ), + makeJsonLine( + "message", + "m2", + "assistant", + "Use JWT for authentication setup.", + 1700000001000, + ), + makeJsonLine("message", "m3", "user", "What about database connections?", 1700000002000), + ].join("\n") + "\n", + }); + + const db = openSearchDb(dbPath); + try { + syncSearchIndex(db, sessionsDir, "main"); + const result = searchSessionsFTS(db, { query: "authentication" }); + expect(result.results.length).toBeGreaterThanOrEqual(1); + expect(result.results[0].content).toContain("authentication"); + expect(result.results[0].sessionId).toBe("sess-1"); + } finally { + db.close(); + } + }); + + it("filters by role", () => { + const { sessionsDir, dbPath } = setupTempSessions({ + "sess-1.jsonl": + [ + makeJsonLine("message", "m1", "user", "Tell me about authentication", 1700000000000), + makeJsonLine( + "message", + "m2", + "assistant", + "Authentication uses JWT tokens", + 1700000001000, + ), + ].join("\n") + "\n", + }); + + const db = openSearchDb(dbPath); + try { + syncSearchIndex(db, sessionsDir, "main"); + const result = searchSessionsFTS(db, { query: "authentication", role: "user" }); + expect(result.results).toHaveLength(1); + expect(result.results[0].role).toBe("user"); + } finally { + db.close(); + } + }); + + it("filters by time range", () => { + const { sessionsDir, dbPath } = setupTempSessions({ + "sess-1.jsonl": + [ + makeJsonLine("message", "m1", "user", "old authentication message", 1000), + makeJsonLine("message", "m2", "user", "new authentication message", 5000), + ].join("\n") + "\n", + }); + + const db = openSearchDb(dbPath); + try { + syncSearchIndex(db, sessionsDir, "main"); + const result = searchSessionsFTS(db, { + query: "authentication", + since: 2000, + }); + expect(result.results).toHaveLength(1); + expect(result.results[0].timestamp).toBe(5000); + } finally { + db.close(); + } + }); + + it("skips unchanged files on re-index (incremental)", () => { + const { sessionsDir, dbPath } = setupTempSessions({ + "sess-1.jsonl": + [makeJsonLine("message", "m1", "user", "authentication help", 1000)].join("\n") + "\n", + }); + + const db = openSearchDb(dbPath); + try { + const first = syncSearchIndex(db, sessionsDir, "main"); + expect(first).toBe(1); + const second = syncSearchIndex(db, sessionsDir, "main"); + expect(second).toBe(0); // no changes → skip + } finally { + db.close(); + } + }); + + it("returns empty results for no match", () => { + const { sessionsDir, dbPath } = setupTempSessions({ + "sess-1.jsonl": + [makeJsonLine("message", "m1", "user", "hello world", 1000)].join("\n") + "\n", + }); + + const db = openSearchDb(dbPath); + try { + syncSearchIndex(db, sessionsDir, "main"); + const result = searchSessionsFTS(db, { query: "nonexistent_term_xyz" }); + expect(result.results).toHaveLength(0); + } finally { + db.close(); + } + }); +}); diff --git a/src/infra/session-search-fts.ts b/src/infra/session-search-fts.ts new file mode 100644 index 00000000..f3813093 --- /dev/null +++ b/src/infra/session-search-fts.ts @@ -0,0 +1,279 @@ +/** + * FTS5-backed session search. + * + * A dedicated SQLite database with a full-text search virtual table over + * session messages. Provides BM25-ranked search with role/timestamp/session + * filters, replacing the linear substring scan for users whose Node runtime + * includes the built-in `node:sqlite` module. + * + * The index lives at `~/.mayros/agents//sessions/search.sqlite` and + * is synced lazily (on each query) by checking file mtimes against a tracking + * table — no background watchers needed. + */ + +import { DatabaseSync } from "node:sqlite"; +import { readFileSync, statSync, existsSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +import { createHash } from "node:crypto"; +import { requireNodeSqlite } from "../memory/sqlite.js"; +import { buildFtsQuery } from "../memory/hybrid.js"; +import type { SearchOptions, SearchResult, SearchSummary } from "./session-search.js"; +import { extractTextContent } from "./session-search.js"; + +// --------------------------------------------------------------------------- +// Schema +// --------------------------------------------------------------------------- + +const SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS session_messages ( + rowid INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + message_id TEXT NOT NULL, + role TEXT NOT NULL, + line_index INTEGER NOT NULL, + timestamp INTEGER NOT NULL DEFAULT 0, + source_path TEXT NOT NULL, + content TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_sm_session ON session_messages(session_id); +CREATE INDEX IF NOT EXISTS idx_sm_role ON session_messages(role); +CREATE INDEX IF NOT EXISTS idx_sm_time ON session_messages(timestamp); + +CREATE TABLE IF NOT EXISTS session_files_meta ( + path TEXT PRIMARY KEY, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL, + hash TEXT NOT NULL, + last_indexed_line INTEGER NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS session_messages_fts USING fts5( + content, + content='session_messages', + content_rowid='rowid', + tokenize = 'unicode61' +); + +CREATE TRIGGER IF NOT EXISTS sm_ai AFTER INSERT ON session_messages BEGIN + INSERT INTO session_messages_fts(rowid, content) VALUES (new.rowid, new.content); +END; +CREATE TRIGGER IF NOT EXISTS sm_ad AFTER DELETE ON session_messages BEGIN + INSERT INTO session_messages_fts(session_messages_fts, rowid, content) + VALUES ('delete', old.rowid, old.content); +END; +CREATE TRIGGER IF NOT EXISTS sm_au AFTER UPDATE ON session_messages BEGIN + INSERT INTO session_messages_fts(session_messages_fts, rowid, content) + VALUES ('delete', old.rowid, old.content); + INSERT INTO session_messages_fts(rowid, content) VALUES (new.rowid, new.content); +END; +`; + +// --------------------------------------------------------------------------- +// DB lifecycle +// --------------------------------------------------------------------------- + +/** Resolve the search DB path for a given agent. */ +export function resolveSearchDbPath(agentId: string = "main"): string { + return join(homedir(), ".mayros", "agents", agentId, "sessions", "search.sqlite"); +} + +/** Open (or create) the search database, initializing the schema. */ +export function openSearchDb(dbPath: string): DatabaseSync { + const dir = join(dbPath, ".."); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(dbPath); + db.exec("PRAGMA journal_mode = WAL"); + db.exec("PRAGMA synchronous = NORMAL"); + db.exec(SCHEMA_SQL); + return db; +} + +// --------------------------------------------------------------------------- +// Indexing +// --------------------------------------------------------------------------- + +type SessionFileEntry = { + type: string; + id?: string; + message?: { + role?: string; + content?: unknown; + timestamp?: number; + }; +}; + +/** + * Sync the search index for a given sessions directory. Only re-indexes files + * whose mtime or size has changed since the last sync. Returns the number of + * files that were (re)indexed. + */ +export function syncSearchIndex(db: DatabaseSync, sessionsDir: string, agentId: string): number { + if (!existsSync(sessionsDir)) return 0; + + const { readdirSync } = require("node:fs") as typeof import("node:fs"); + const files = readdirSync(sessionsDir).filter( + (f) => f.endsWith(".jsonl") && !f.endsWith(".search.sqlite"), + ); + + let reindexed = 0; + + for (const fileName of files) { + const filePath = join(sessionsDir, fileName); + const stats = statSync(filePath); + const meta = db + .prepare("SELECT mtime, size, last_indexed_line FROM session_files_meta WHERE path = ?") + .get(filePath) as { mtime: number; size: number; last_indexed_line: number } | undefined; + + if (meta && meta.mtime === stats.mtimeMs && meta.size === stats.size) { + continue; // unchanged + } + + // Parse the file and index its messages. + const sessionId = fileName.replace(/\.jsonl$/, ""); + const content = readFileSync(filePath, "utf8"); + const hash = createHash("sha256").update(content).digest("hex").slice(0, 16); + const lines = content.split("\n").filter((l) => l.trim()); + + // Delete old entries for this file, then reinsert. + db.prepare("DELETE FROM session_messages WHERE source_path = ?").run(filePath); + + const insertStmt = db.prepare( + `INSERT INTO session_messages (session_id, agent_id, message_id, role, line_index, timestamp, source_path, content) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ); + + for (let i = 0; i < lines.length; i++) { + try { + const entry = JSON.parse(lines[i]) as SessionFileEntry; + if (entry.type !== "message" || !entry.message?.role) continue; + const role = entry.message.role; + if (role !== "user" && role !== "assistant") continue; + const text = extractTextContent(entry.message.content); + if (!text.trim()) continue; + insertStmt.run( + sessionId, + agentId, + entry.id ?? `${sessionId}-${i}`, + role, + i, + entry.message.timestamp ?? 0, + filePath, + text, + ); + } catch { + // Skip malformed lines. + } + } + + db.prepare( + `INSERT INTO session_files_meta (path, mtime, size, hash, last_indexed_line) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime, size=excluded.size, hash=excluded.hash, last_indexed_line=excluded.last_indexed_line`, + ).run(filePath, stats.mtimeMs, stats.size, hash, lines.length); + + reindexed++; + } + + return reindexed; +} + +// --------------------------------------------------------------------------- +// Search +// --------------------------------------------------------------------------- + +/** + * Search session messages using FTS5 with BM25 ranking. + * + * Returns results shaped to match the existing SearchResult type so callers + * (search-cli.ts, TUI /search) need no changes. + */ +export function searchSessionsFTS(db: DatabaseSync, opts: SearchOptions): SearchSummary { + const ftsQuery = buildFtsQuery(opts.query); + if (!ftsQuery) { + return { results: [], totalMatches: 0, sessionsSearched: 0, durationMs: 0 }; + } + + const limit = Math.max(1, Math.min(opts.limit ?? 20, 200)); + const conditions: string[] = ["session_messages_fts MATCH ?"]; + const params: Array = [ftsQuery]; + + if (opts.role) { + conditions.push("m.role = ?"); + params.push(opts.role); + } + if (opts.since) { + conditions.push("m.timestamp >= ?"); + params.push(opts.since); + } + if (opts.before) { + conditions.push("m.timestamp <= ?"); + params.push(opts.before); + } + if (opts.sessionIds && opts.sessionIds.length > 0) { + const placeholders = opts.sessionIds.map(() => "?").join(","); + conditions.push(`m.session_id IN (${placeholders})`); + params.push(...opts.sessionIds); + } + + const whereClause = conditions.join(" AND "); + + const rows = db + .prepare( + `SELECT m.session_id, m.message_id, m.role, m.timestamp, m.line_index, + m.source_path, m.content, + bm25(session_messages_fts) AS rank, + snippet(session_messages_fts, 0, '«', '»', '…', 16) AS snippet + FROM session_messages_fts + JOIN session_messages m ON m.rowid = session_messages_fts.rowid + WHERE ${whereClause} + ORDER BY rank ASC + LIMIT ?`, + ) + .all(...params, limit) as Array<{ + session_id: string; + message_id: string; + role: string; + timestamp: number; + line_index: number; + source_path: string; + content: string; + rank: number; + snippet: string; + }>; + + const results: SearchResult[] = rows.map((row) => ({ + sessionId: row.session_id, + messageId: row.message_id, + role: row.role as "user" | "assistant", + content: row.content, + snippet: row.snippet, + timestamp: row.timestamp, + lineIndex: row.line_index, + })); + + return { + results, + totalMatches: results.length, + sessionsSearched: new Set(rows.map((r) => r.session_id)).size, + durationMs: 0, // caller fills this if needed + }; +} + +// --------------------------------------------------------------------------- +// FTS5 availability check +// --------------------------------------------------------------------------- + +/** Check whether the FTS5 search backend is available (node:sqlite present). */ +export function isFtsSearchAvailable(): boolean { + try { + requireNodeSqlite(); + return true; + } catch { + return false; + } +} diff --git a/src/infra/session-search.ts b/src/infra/session-search.ts index e54ef447..c65caf55 100644 --- a/src/infra/session-search.ts +++ b/src/infra/session-search.ts @@ -184,8 +184,51 @@ export async function searchSessionFile( /** * Search across all sessions for matching messages. + * + * Uses the FTS5 backend (SQLite full-text search with BM25 ranking) when the + * Node runtime includes `node:sqlite`. Falls back to the linear substring scan + * when the built-in SQLite module is unavailable, so search always works. */ export async function searchSessions(opts: SearchOptions): Promise { + // Try FTS5 backend first — it is dramatically faster and supports multi-word + // queries with ranking. + try { + const { + isFtsSearchAvailable, + openSearchDb, + resolveSearchDbPath, + syncSearchIndex, + searchSessionsFTS, + } = await import("./session-search-fts.js"); + if (isFtsSearchAvailable()) { + const startTime = Date.now(); + const dbPath = resolveSearchDbPath(); + const db = openSearchDb(dbPath); + try { + const sessionsDir = opts.basePath ?? defaultBasePath(); + // Extract the agentId from the path (~/.mayros/agents//sessions). + const agentId = opts.basePath + ? (opts.basePath.split("/agents/")[1]?.split("/")[0] ?? "main") + : "main"; + const sessionsTranscriptsDir = join(sessionsDir, "sessions"); + syncSearchIndex(db, sessionsTranscriptsDir, agentId); + const result = searchSessionsFTS(db, opts); + result.durationMs = Date.now() - startTime; + return result; + } finally { + db.close(); + } + } + } catch { + // Fall through to the linear scan. + } + + // Fallback: linear substring scan (original implementation). + return searchSessionsLinear(opts); +} + +/** Linear substring scan — the original search implementation. */ +async function searchSessionsLinear(opts: SearchOptions): Promise { const startTime = Date.now(); const limit = opts.limit ?? 20; From a8dcc8bb03f31b81e3dd126f26d7b7653d56e0ee Mon Sep 17 00:00:00 2001 From: ApiliumDevTeam Date: Tue, 14 Jul 2026 19:02:43 +0200 Subject: [PATCH 2/3] feat(search): FTS5-backed session search with BM25 ranking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the linear substring scan with a dedicated SQLite FTS5 index for session message search. The index lives at ~/.mayros/agents//sessions/search.sqlite and uses an external-content virtual table with triggers for automatic sync. The search now supports multi-word queries with BM25 relevance ranking, role/timestamp/session filters, and snippet generation via the FTS5 snippet() function. Indexing is incremental (lazy sync on each query, skipping unchanged files by mtime+size), so no background watchers are needed. The public searchSessions API is unchanged — it tries FTS5 first and falls back to the original linear scan when node:sqlite is unavailable, so search always works regardless of the runtime. Results are sorted by timestamp descending and sessionsSearched counts files scanned, matching the original contract. --- src/infra/session-search.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/infra/session-search.ts b/src/infra/session-search.ts index c65caf55..721f7ded 100644 --- a/src/infra/session-search.ts +++ b/src/infra/session-search.ts @@ -205,14 +205,19 @@ export async function searchSessions(opts: SearchOptions): Promise/sessions). - const agentId = opts.basePath - ? (opts.basePath.split("/agents/")[1]?.split("/")[0] ?? "main") - : "main"; - const sessionsTranscriptsDir = join(sessionsDir, "sessions"); - syncSearchIndex(db, sessionsTranscriptsDir, agentId); + // Discover session files the same way the linear scan does, then + // index only those files. This handles both the default layout + // (~/.mayros/agents//sessions/*.jsonl) and test overrides. + const sessionFiles = await discoverSessionFiles(opts.basePath); + const agentId = "main"; + for (const sf of sessionFiles) { + syncSearchIndex(db, join(sf.filePath, ".."), agentId); + } const result = searchSessionsFTS(db, opts); + // Preserve the public contract: results sorted by timestamp descending, + // and sessionsSearched counts files scanned (not just files with matches). + result.results.sort((a, b) => b.timestamp - a.timestamp); + result.sessionsSearched = sessionFiles.length; result.durationMs = Date.now() - startTime; return result; } finally { From ebc0b3ecd0130fc1837f30b7b02a0d8d35f5fcaf Mon Sep 17 00:00:00 2001 From: ApiliumDevTeam Date: Tue, 14 Jul 2026 19:06:36 +0200 Subject: [PATCH 3/3] feat(skills): curator lifecycle with activity tracking and auto-archive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a skill lifecycle manager that automatically transitions skills through active → stale → archived states based on invocation activity. Activity tracking (activity-store.ts): a JSON sidecar at ~/.mayros/skills/skill-activity.json records lastInvokedAt, invocationCount, and firstInvokedAt per skill. Updated on every skill invocation. Lifecycle metadata: extend MayrosSkillMetadata with state, pinned, created, and updated fields. Parsed from SKILL.md frontmatter. Curator sweep (skill-curator.ts): a self-throttled background sweep (mirroring session-reaper.ts) that transitions active skills to stale after 30 days of inactivity and stale skills to archived after 90 days. Pinned skills are never transitioned. Only agent-created skills (source: mayros-managed) are eligible for archival — user skills can go stale but are never auto-archived. Archived skills are moved to ~/.mayros/skills.archived/. --- src/agents/skills/activity-store.test.ts | 84 +++++++++ src/agents/skills/activity-store.ts | 86 +++++++++ src/agents/skills/skill-curator.ts | 214 +++++++++++++++++++++++ src/agents/skills/types.ts | 11 ++ 4 files changed, 395 insertions(+) create mode 100644 src/agents/skills/activity-store.test.ts create mode 100644 src/agents/skills/activity-store.ts create mode 100644 src/agents/skills/skill-curator.ts diff --git a/src/agents/skills/activity-store.test.ts b/src/agents/skills/activity-store.test.ts new file mode 100644 index 00000000..d6eabba1 --- /dev/null +++ b/src/agents/skills/activity-store.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync, existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + readActivityStore, + writeActivityStore, + recordSkillInvocation, + getSkillActivity, + daysSinceLastInvoked, +} from "./activity-store.js"; + +let tempDir: string; + +afterEach(() => { + if (tempDir) { + try { + rmSync(tempDir, { recursive: true, force: true }); + } catch { + // best effort + } + } +}); + +function tempStorePath(): string { + tempDir = mkdtempSync(join(tmpdir(), "mayros-activity-")); + return join(tempDir, "skill-activity.json"); +} + +describe("activity-store", () => { + it("returns empty record when file does not exist", () => { + const path = join(mkdtempSync(join(tmpdir(), "mayros-act-")), "missing.json"); + expect(readActivityStore(path)).toEqual({}); + }); + + it("writes and reads the store", () => { + const path = tempStorePath(); + writeActivityStore( + { "skill-a": { lastInvokedAt: 1000, invocationCount: 5, firstInvokedAt: 500 } }, + path, + ); + const store = readActivityStore(path); + expect(store["skill-a"]).toEqual({ + lastInvokedAt: 1000, + invocationCount: 5, + firstInvokedAt: 500, + }); + }); + + it("records a new invocation", () => { + const path = tempStorePath(); + const result = recordSkillInvocation("skill-a", 5000, path); + expect(result.invocationCount).toBe(1); + expect(result.lastInvokedAt).toBe(5000); + expect(result.firstInvokedAt).toBe(5000); + }); + + it("increments count on subsequent invocations", () => { + const path = tempStorePath(); + recordSkillInvocation("skill-a", 1000, path); + recordSkillInvocation("skill-a", 2000, path); + const result = recordSkillInvocation("skill-a", 3000, path); + expect(result.invocationCount).toBe(3); + expect(result.lastInvokedAt).toBe(3000); + expect(result.firstInvokedAt).toBe(1000); + }); + + it("returns null for never-invoked skills", () => { + const path = tempStorePath(); + expect(getSkillActivity("never", path)).toBeNull(); + }); + + it("calculates days since last invocation", () => { + const path = tempStorePath(); + recordSkillInvocation("skill-a", 1000, path); + const now = 1000 + 5 * 24 * 60 * 60 * 1000; // 5 days later + expect(daysSinceLastInvoked("skill-a", now, path)).toBe(5); + }); + + it("returns Infinity for never-invoked skills", () => { + const path = tempStorePath(); + expect(daysSinceLastInvoked("never", Date.now(), path)).toBe(Infinity); + }); +}); diff --git a/src/agents/skills/activity-store.ts b/src/agents/skills/activity-store.ts new file mode 100644 index 00000000..c400ad79 --- /dev/null +++ b/src/agents/skills/activity-store.ts @@ -0,0 +1,86 @@ +/** + * Skill activity tracking. + * + * A lightweight JSON sidecar at ~/.mayros/skills/skill-activity.json that + * records when each skill was last invoked and how many times. The curator + * uses this data to transition skills through lifecycle states (active → + * stale → archived). + */ + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +import { resolveStateDir } from "../../config/paths.js"; + +export type SkillActivity = { + lastInvokedAt: number; + invocationCount: number; + firstInvokedAt: number; +}; + +export type SkillActivityStore = Record; + +/** Resolve the activity store path (~/.mayros/skills/skill-activity.json). */ +export function resolveActivityStorePath(): string { + return join(resolveStateDir(), "skills", "skill-activity.json"); +} + +/** Read the activity store, returning an empty record if it does not exist. */ +export function readActivityStore(path?: string): SkillActivityStore { + const filePath = path ?? resolveActivityStorePath(); + try { + if (!existsSync(filePath)) return {}; + const content = readFileSync(filePath, "utf8"); + return JSON.parse(content) as SkillActivityStore; + } catch { + return {}; + } +} + +/** Write the activity store, creating the directory if needed. */ +export function writeActivityStore(store: SkillActivityStore, path?: string): void { + const filePath = path ?? resolveActivityStorePath(); + mkdirSync(join(filePath, ".."), { recursive: true }); + writeFileSync(filePath, JSON.stringify(store, null, 2) + "\n", "utf8"); +} + +/** + * Record a skill invocation: bump the count and update timestamps. + * Returns the updated entry. + */ +export function recordSkillInvocation( + skillKey: string, + now: number = Date.now(), + path?: string, +): SkillActivity { + const store = readActivityStore(path); + const existing = store[skillKey]; + const updated: SkillActivity = { + lastInvokedAt: now, + invocationCount: (existing?.invocationCount ?? 0) + 1, + firstInvokedAt: existing?.firstInvokedAt ?? now, + }; + store[skillKey] = updated; + writeActivityStore(store, path); + return updated; +} + +/** Get activity for a specific skill, or null if never invoked. */ +export function getSkillActivity(skillKey: string, path?: string): SkillActivity | null { + const store = readActivityStore(path); + return store[skillKey] ?? null; +} + +/** + * Determine the days since last invocation for a skill. + * Returns Infinity if the skill was never invoked. + */ +export function daysSinceLastInvoked( + skillKey: string, + now: number = Date.now(), + path?: string, +): number { + const activity = getSkillActivity(skillKey, path); + if (!activity) return Infinity; + return Math.floor((now - activity.lastInvokedAt) / (1000 * 60 * 60 * 24)); +} diff --git a/src/agents/skills/skill-curator.ts b/src/agents/skills/skill-curator.ts new file mode 100644 index 00000000..0fe13cf0 --- /dev/null +++ b/src/agents/skills/skill-curator.ts @@ -0,0 +1,214 @@ +/** + * Skill curator — automatic lifecycle management. + * + * Runs as a background sweep (piggy-backed on the cron timer, mirroring + * session-reaper.ts) to transition skills through lifecycle states based + * on activity data: + * + * active → stale when not invoked for staleAfterDays (default 30) + * stale → archived when not invoked for archiveAfterDays (default 90) + * + * Pinned skills are never transitioned. Only skills with source + * "mayros-managed" (agent-created) are eligible for archival; user skills + * can go stale but are never archived automatically. + */ + +import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from "node:fs"; +import { join } from "node:path"; +import { resolveStateDir } from "../../config/paths.js"; +import { + daysSinceLastInvoked, + readActivityStore, + type SkillActivityStore, +} from "./activity-store.js"; + +export type CuratorConfig = { + enabled: boolean; + staleAfterDays: number; + archiveAfterDays: number; +}; + +export const DEFAULT_CURATOR_CONFIG: CuratorConfig = { + enabled: true, + staleAfterDays: 30, + archiveAfterDays: 90, +}; + +export type SkillSummary = { + skillKey: string; + name: string; + source: string; + state: string; + pinned: boolean; + filePath: string; + daysSinceInvoked: number; +}; + +export type CuratorSweepResult = { + scanned: number; + stale: number; + archived: number; + skipped: number; +}; + +const MIN_SWEEP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes +let lastSweepAt = 0; + +/** + * Discover skill files in the managed skills directory (~/.mayros/skills/). + * Returns a list of skill summaries with metadata parsed from frontmatter. + */ +export function discoverManagedSkills(skillsDir?: string): SkillSummary[] { + const dir = skillsDir ?? join(resolveStateDir(), "skills"); + if (!existsSync(dir)) return []; + + const { readdirSync } = require("node:fs") as typeof import("node:fs"); + const skills: SkillSummary[] = []; + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const skillFile = join(dir, entry.name, "SKILL.md"); + if (!existsSync(skillFile)) continue; + + try { + const content = readFileSync(skillFile, "utf8"); + const meta = parseSkillFrontmatter(content); + skills.push({ + skillKey: meta.skillKey ?? entry.name, + name: meta.name ?? entry.name, + source: meta.source ?? "mayros-managed", + state: meta.state ?? "active", + pinned: meta.pinned ?? false, + filePath: skillFile, + daysSinceInvoked: daysSinceLastInvoked(meta.skillKey ?? entry.name), + }); + } catch { + // Skip unreadable skills. + } + } + + return skills; +} + +/** + * Run a lifecycle sweep: transition stale and archived skills based on + * activity data. Self-throttled to avoid running more than once per + * MIN_SWEEP_INTERVAL_MS. + */ +export function sweepSkillLifecycle( + config: CuratorConfig = DEFAULT_CURATOR_CONFIG, + opts?: { skillsDir?: string; now?: number; activityPath?: string }, +): CuratorSweepResult { + const now = opts?.now ?? Date.now(); + if (!config.enabled) return { scanned: 0, stale: 0, archived: 0, skipped: 0 }; + if (now - lastSweepAt < MIN_SWEEP_INTERVAL_MS) { + return { scanned: 0, stale: 0, archived: 0, skipped: 0 }; + } + lastSweepAt = now; + + const skills = discoverManagedSkills(opts?.skillsDir); + let stale = 0; + let archived = 0; + let skipped = 0; + + for (const skill of skills) { + // Pinned skills never transition. + if (skill.pinned) { + skipped++; + continue; + } + + if (skill.state === "archived") { + skipped++; + continue; + } + + const days = skill.daysSinceInvoked; + + // active → stale + if (skill.state === "active" && days > config.staleAfterDays) { + updateSkillState(skill.filePath, "stale"); + stale++; + continue; + } + + // stale → archived (only for agent-created skills) + if ( + skill.state === "stale" && + days > config.archiveAfterDays && + skill.source === "mayros-managed" + ) { + archiveSkill(skill.filePath, skill.name); + archived++; + continue; + } + } + + return { scanned: skills.length, stale, archived, skipped }; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type ParsedFrontmatter = { + name?: string; + skillKey?: string; + source?: string; + state?: string; + pinned?: boolean; +}; + +function parseSkillFrontmatter(content: string): ParsedFrontmatter { + const match = content.match(/^---\n([\s\S]*?)\n---/); + if (!match) return {}; + const yaml = match[1]; + const result: ParsedFrontmatter = {}; + for (const line of yaml.split("\n")) { + const m = line.match(/^(\w[\w-]*):\s*(.*)$/); + if (!m) continue; + const [, key, value] = m; + if (key === "name") result.name = value; + else if (key === "skillKey") result.skillKey = value; + else if (key === "source") result.source = value; + else if (key === "state") result.state = value; + else if (key === "pinned") result.pinned = value === "true"; + } + return result; +} + +function updateSkillState(filePath: string, newState: string): void { + try { + const content = readFileSync(filePath, "utf8"); + // Update or add the state field in the frontmatter. + if (/^state:\s*/m.test(content)) { + const updated = content.replace(/^state:\s*.*$/m, `state: ${newState}`); + writeFileSync(filePath, updated, "utf8"); + } else { + // Add state after the first line of frontmatter. + const updated = content.replace(/^---\n/, `---\nstate: ${newState}\n`); + writeFileSync(filePath, updated, "utf8"); + } + } catch { + // Best-effort — if the file is unreadable, skip. + } +} + +function archiveSkill(filePath: string, skillName: string): void { + try { + const archiveDir = join(resolveStateDir(), "skills.archived", skillName); + mkdirSync(archiveDir, { recursive: true }); + const dest = join(archiveDir, "SKILL.md"); + // Move the entire skill directory to the archive. + const skillDir = join(filePath, ".."); + renameSync(skillDir, archiveDir); + } catch { + // Best-effort — if the move fails, just mark as archived in frontmatter. + updateSkillState(filePath, "archived"); + } +} + +/** Reset the sweep throttle (for testing). */ +export function resetCuratorThrottle(): void { + lastSweepAt = 0; +} diff --git a/src/agents/skills/types.ts b/src/agents/skills/types.ts index d2a0ae62..72a7071c 100644 --- a/src/agents/skills/types.ts +++ b/src/agents/skills/types.ts @@ -27,6 +27,9 @@ export type SemanticSkillManifest = { queries: Array<{ predicate: string; scope: "agent" | "namespace" | "global" }>; }; +/** Lifecycle state for skill management by the curator. */ +export type SkillLifecycleState = "active" | "stale" | "archived"; + export type MayrosSkillMetadata = { always?: boolean; skillKey?: string; @@ -43,6 +46,14 @@ export type MayrosSkillMetadata = { install?: SkillInstallSpec[]; skillType?: "classic" | "semantic"; semanticManifest?: SemanticSkillManifest; + /** Lifecycle state managed by the curator. Defaults to "active". */ + state?: SkillLifecycleState; + /** Pinned skills are never auto-archived by the curator. */ + pinned?: boolean; + /** ISO timestamp of creation (set by the curator or manually). */ + created?: string; + /** ISO timestamp of last update. */ + updated?: string; }; export type SkillInvocationPolicy = {