From ec31904db383f10e51e7c4061d9c32602e5f3c54 Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:21:39 +0100 Subject: [PATCH] feat(memory): add relevance-based auto-recall and opt-in dreaming Closes #460 --- .github/i18n/README.de.md | 27 +- README.md | 14 +- opencode-mem.example.jsonc | 17 + src/config.ts | 99 +++++ src/index.ts | 39 +- src/services/api-handlers.ts | 139 +++++- src/services/cleanup-service.ts | 19 +- src/services/client.ts | 36 +- src/services/dreaming-lock.ts | 87 ++++ src/services/dreaming-service.ts | 395 ++++++++++++++++++ src/services/memory-recall.ts | 244 +++++++++++ src/services/stats-service.ts | 8 +- src/services/turso/schema-migrations.ts | 8 + src/services/turso/types.ts | 3 + src/services/turso/vector-search.ts | 109 ++++- .../user-prompt/user-prompt-manager.ts | 17 + src/services/web-server.ts | 34 +- tests/config.test.ts | 12 + tests/dreaming-prompt.test.ts | 40 ++ tests/fixtures/child-session-capture.mjs | 5 + .../fixtures/memory-injection-eligibility.mjs | 5 + .../profile-inject-empty-memories.mjs | 5 + tests/fixtures/v2-memory-context.mjs | 5 + tests/markdown-write-through.test.ts | 6 +- tests/memory-recall.test.ts | 78 ++++ tests/memory-trash.test.ts | 129 ++++++ tests/profile-inject-empty-memories.test.ts | 3 +- tests/schema-migrations.test.ts | 11 +- web/src/App.svelte | 13 + .../components/explorer/TrashDialog.svelte | 91 ++++ web/src/lib/i18n/translations.ts | 95 +++++ .../lib/stores/memories-explorer.svelte.ts | 69 +++ 32 files changed, 1769 insertions(+), 93 deletions(-) create mode 100644 src/services/dreaming-lock.ts create mode 100644 src/services/dreaming-service.ts create mode 100644 src/services/memory-recall.ts create mode 100644 tests/dreaming-prompt.test.ts create mode 100644 tests/memory-recall.test.ts create mode 100644 tests/memory-trash.test.ts create mode 100644 web/src/lib/components/explorer/TrashDialog.svelte diff --git a/.github/i18n/README.de.md b/.github/i18n/README.de.md index 54e50466..7561ed56 100644 --- a/.github/i18n/README.de.md +++ b/.github/i18n/README.de.md @@ -262,14 +262,16 @@ Wenn Markdown-Modus aktiv ist: Diese Einstellungen liegen unter dem Schlüssel `chatMessage` in `~/.config/opencode/opencode-mem.jsonc`: -| Option | Standard | Was sie tut | -| ------------------------------------------------------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | `true` | Verfasste User-Prompts erfassen und Speicher-Kontext injizieren. | -| `injectOn` | `"first"` | Speicher-Kontext bei der ersten User-Nachricht einer Sitzung injizieren, oder bei `"always"` bei jedem verfassten Turn. | -| `maxMemories` / `excludeCurrentSession` / `maxAgeDays` | `3` / `true` / — | Wie viele Memories der injizierte Kontext hält, ob Memories aus der aktuellen Sitzung ausgeschlossen werden, und optionaler Alters-Cutoff in Tagen. | -| `filterInjectedPrompts` | `true` | Prompt-Text überspringen, der vom Host oder anderen OpenCode-Plugins injiziert wurde (System-Reminders, Orchestrierungsanweisungen, Background-Task-Benachrichtigungen), damit er nie so gespeichert wird, als hätte ihn der User getippt. | -| `injectionMarkers` | Built-ins | Zusätzliche Marker zur Erkennung injizierter Blöcke; werden zur Built-in-Liste hinzugefügt, ersetzen sie nie. | -| `captureChildSessions` | `false` | Prompts aus Orchestrator-Child-Sitzungen erfassen (Sitzungen mit `parentID`, z. B. OpenCode-Task-/Subagent-Children). Deren „User“-Nachrichten schreibt der Parent-Agent, nicht du — daher werden sie standardmäßig nicht gespeichert, auto-captured, für Profil-Lernen genutzt oder mit injiziertem Speicher-Kontext versehen. | +| Option | Standard | Was sie tut | +| ------------------------------------------------------ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `true` | Verfasste User-Prompts erfassen und Speicher-Kontext injizieren. | +| `injectOn` | `"first"` | Bei `recall: "recent"`: Kontext bei der ersten User-Nachricht oder bei `"always"` jedem Turn. Bei `recall: "relevant"` / `"both"` wird der Kontext bei jedem verfassten Turn neu geladen (Issue #460). | +| `recall` | `"relevant"` | Auswahl injizierter Memories: `"relevant"` bettet die User-Nachricht ein und sucht per Vektor (jeder Turn); `"recent"` behält die Legacy-Newest-N-Liste; `"both"` bevorzugt relevant und füllt mit recent auf. | +| `recallScope` | `"default"` | Scope für Auto-Recall: `"default"` folgt `memory.defaultScope`, oder `"project"` / `"all-projects"`. Bei All-Projects erhalten Treffer aus dem aktuellen Projekt einen kleinen Similarity-Boost. | +| `maxMemories` / `excludeCurrentSession` / `maxAgeDays` | `3` / `true` / — | Wie viele Memories der injizierte Kontext hält, ob Memories aus der aktuellen Sitzung ausgeschlossen werden, und optionaler Alters-Cutoff in Tagen. | +| `filterInjectedPrompts` | `true` | Prompt-Text überspringen, der vom Host oder anderen OpenCode-Plugins injiziert wurde (System-Reminders, Orchestrierungsanweisungen, Background-Task-Benachrichtigungen), damit er nie so gespeichert wird, als hätte ihn der User getippt. | +| `injectionMarkers` | Built-ins | Zusätzliche Marker zur Erkennung injizierter Blöcke; werden zur Built-in-Liste hinzugefügt, ersetzen sie nie. | +| `captureChildSessions` | `false` | Prompts aus Orchestrator-Child-Sitzungen erfassen (Sitzungen mit `parentID`, z. B. OpenCode-Task-/Subagent-Children). Deren „User“-Nachrichten schreibt der Parent-Agent, nicht du — daher werden sie standardmäßig nicht gespeichert, auto-captured, für Profil-Lernen genutzt oder mit injiziertem Speicher-Kontext versehen. | Ausschlüsse nach Sitzung und Alter werden vor dem Limit für injizierte Memories angewendet. Ausgeschlossene Memories belegen daher keine Plätze, die für geeignete @@ -277,6 +279,15 @@ Memories verfügbar wären. Mit `chatMessage.maxMemories` auf `0` wird die Injek von Projekt-Memories deaktiviert; das Nutzerprofil kann weiterhin injiziert werden, wenn es aktiviert ist. +### Dreaming (optionale Memory-Konsolidierung) + +Opt-in-Hintergrundlauf, der Near-Duplikate findet (gleicher Similarity-Schwellwert wie +Deduplizierung), sie über `opencodeProvider`/`opencodeModel` zusammenführt und ersetzte +Einträge soft-deleten (wiederherstellbarer Papierkorb). Gepinnte Memories bleiben unberührt. +Konfiguration unter `dreaming` in `opencode-mem.jsonc` (`enabled` Standard `false`). Die +Web-UI bietet **Papierkorb** und **Dreaming**. Soft-gelöschte Zeilen werden nach +`dreaming.trashRetentionDays` beim Cleanup hart bereinigt. + ### Embeddings wählen / konfigurieren Embeddings treiben die Ähnlichkeitssuche für Memories und das Benutzerprofil. Konfiguration in derselben Datei (`~/.config/opencode/opencode-mem.jsonc`). Es gibt **kein MLX-Backend** — lokale Embeddings nutzen `@huggingface/transformers` mit ONNX, nicht Apple MLX. diff --git a/README.md b/README.md index 61aed6ce..6515cab0 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,10 @@ These settings live under the `chatMessage` key in `~/.config/opencode/opencode- | Option | Default | What it does | | ------------------------------------------------------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `enabled` | `true` | Capture authored user prompts and inject memory context. | -| `injectOn` | `"first"` | Inject memory context on the first user message of a session, or on `"always"` every authored turn. | +| `injectOn` | `"first"` | For `recall: "recent"`, inject on the first user message or on `"always"` every authored turn. With `recall: "relevant"` / `"both"`, context always refreshes each authored turn (issue #460). | +| `recall` | `"relevant"` | How injected memories are chosen: `"relevant"` embeds the user message and runs vector search each turn; `"recent"` keeps the legacy newest-N listing; `"both"` prefers relevant then fills remaining slots with recent. | +| `recallScope` | `"default"` | Scope for auto-recall: `"default"` follows `memory.defaultScope`, or force `"project"` / `"all-projects"`. When searching all projects, hits from the current project get a small similarity boost. | +| `similarityThreshold` | (top-level) | Optional per-inject floor; otherwise uses the top-level `similarityThreshold`. | | `maxMemories` / `excludeCurrentSession` / `maxAgeDays` | `3` / `true` / — | How many memories the injected context holds, whether memories captured in the current session are excluded, and an optional age cutoff in days. | | `filterInjectedPrompts` | `true` | Skip prompt text injected by the host or by other OpenCode plugins (system reminders, orchestration directives, background-task notifications) so it is never stored as if the user had typed it. | | `injectionMarkers` | built-ins | Extra markers identifying injected blocks; added to the built-in list, never replacing it. | @@ -276,6 +279,15 @@ memories do not consume slots that could hold eligible memories. Setting `chatMessage.maxMemories` to `0` disables project-memory injection while still allowing the user profile to be injected when enabled. +### Dreaming (optional memory consolidation) + +Opt-in background pass that finds near-duplicate memories (same similarity threshold as +deduplication), asks the configured `opencodeProvider` / `opencodeModel` to merge them, +and soft-deletes superseded rows into recoverable trash. Pinned memories are never +touched. Configure under `dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). +The web UI exposes **Trash** (restore) and **Dream** (manual run). Soft-deleted rows are +hard-purged after `dreaming.trashRetentionDays` during normal cleanup. + ### Choosing / configuring embeddings Embeddings power similarity search for memories and the user profile. Configure them in the same file (`~/.config/opencode/opencode-mem.jsonc`). There is **no MLX backend** — local embeddings use `@huggingface/transformers` with ONNX, not Apple MLX. diff --git a/opencode-mem.example.jsonc b/opencode-mem.example.jsonc index 382c240a..801c7027 100644 --- a/opencode-mem.example.jsonc +++ b/opencode-mem.example.jsonc @@ -131,6 +131,14 @@ // "maxAgeDays": 30, // "first" = only on the first user message in a session; "always" = every message "injectOn": "first", + // "relevant" (default) = vector search against each authored user prompt + // (refreshes every turn even when injectOn is "first"); + // "recent" = newest-N listing (honors injectOn); "both" = relevant first, then fill + "recall": "relevant", + // "default" follows memory.defaultScope; or "project" / "all-projects" + "recallScope": "default", + // Optional override of top-level similarityThreshold for relevant recall: + // "similarityThreshold": 0.6, // Skip prompt text injected by the host or by other OpenCode plugins // (system reminders, orchestration directives, background-task // notifications) so it is never stored as if the user had typed it. @@ -147,6 +155,15 @@ // "injectionMarkers": [""], }, + // Opt-in background consolidation ("dreaming"): merge near-duplicates via LLM + // and soft-delete superseded memories into recoverable trash. + "dreaming": { + "enabled": false, + "intervalHours": 24, + "maxGroupsPerRun": 5, + "trashRetentionDays": 30, + }, + // ============================================ // OpenCode Provider Settings (RECOMMENDED) // ============================================ diff --git a/src/config.ts b/src/config.ts index d518f0a0..2917e83b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -207,6 +207,19 @@ interface OpenCodeMemConfig { excludeCurrentSession?: boolean; maxAgeDays?: number; injectOn?: "first" | "always"; + /** + * How auto-injected memories are selected. + * - recent: newest pinned/recency listing (legacy) + * - relevant: vector search against the current user prompt (default) + * - both: relevant first, then fill remaining slots with recent + */ + recall?: "recent" | "relevant" | "both"; + /** + * Scope for auto-recall search/list. "default" follows memory.defaultScope. + */ + recallScope?: "project" | "all-projects" | "default"; + /** Optional override of top-level similarityThreshold for relevant recall. */ + similarityThreshold?: number; filterInjectedPrompts?: boolean; injectionMarkers?: string[]; /** @@ -217,6 +230,16 @@ interface OpenCodeMemConfig { */ captureChildSessions?: boolean; }; + /** + * Opt-in background memory consolidation ("dreaming"): merge near-duplicate + * memories via the configured LLM and soft-delete superseded rows into trash. + */ + dreaming?: { + enabled?: boolean; + intervalHours?: number; + maxGroupsPerRun?: number; + trashRetentionDays?: number; + }; } const DEFAULTS: Required< @@ -354,10 +377,19 @@ const DEFAULTS: Required< excludeCurrentSession: true, maxAgeDays: undefined, injectOn: "first", + recall: "relevant", + recallScope: "default", + similarityThreshold: undefined, filterInjectedPrompts: true, injectionMarkers: [...getDefaultInjectionMarkers()], captureChildSessions: false, }, + dreaming: { + enabled: false, + intervalHours: 24, + maxGroupsPerRun: 5, + trashRetentionDays: 30, + }, }; function expandPath(path: string): string { @@ -830,6 +862,16 @@ const CONFIG_TEMPLATE = `{ // Chat message capture and injection behavior "chatMessage": { + // How auto-injected memories are picked for the prompt context. + // "relevant" (default) embeds the user message and runs vector search. + // "recent" keeps the legacy newest-N listing. "both" prefers relevant, + // then fills remaining slots with recent memories. + "recall": "relevant", + // Scope for auto-recall. "default" follows memory.defaultScope above. + // "recallScope": "default", + // Optional similarity floor for relevant recall (else similarityThreshold). + // "similarityThreshold": 0.6, + // Skip prompt text injected by the host or by other OpenCode plugins // (system reminders, orchestration directives, background-task // notifications) so it is never stored as if the user had typed it. @@ -846,6 +888,15 @@ const CONFIG_TEMPLATE = `{ // Extra markers identifying injected blocks. These are ADDED to the // built-in list, never replace it. // "injectionMarkers": [""] + }, + + // Opt-in background consolidation: merge near-duplicate memories via the + // configured LLM and soft-delete superseded rows into trash (recoverable). + "dreaming": { + "enabled": false, + "intervalHours": 24, + "maxGroupsPerRun": 5, + "trashRetentionDays": 30 } } `; @@ -1074,6 +1125,39 @@ export function outerStructuredOutputTimeoutMs(configured: number | undefined): return (configured ?? OPENCODE_TIMEOUT_MS_DEFAULT) + OPENCODE_TIMEOUT_OUTER_MARGIN_MS; } +export function normalizeChatMessageRecall(value: unknown): "recent" | "relevant" | "both" { + if (value === "recent" || value === "relevant" || value === "both") return value; + return "relevant"; +} + +export function normalizeChatMessageRecallScope( + value: unknown +): "project" | "all-projects" | "default" { + if (value === "project" || value === "all-projects" || value === "default") return value; + return "default"; +} + +export function normalizeDreamingIntervalHours(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return 24; + } + return Math.max(1, Math.trunc(value)); +} + +export function normalizeDreamingMaxGroupsPerRun(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return 5; + } + return Math.max(1, Math.trunc(value)); +} + +export function normalizeDreamingTrashRetentionDays(value: unknown): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) { + return 30; + } + return value; +} + /** * User-supplied markers extend the built-in set rather than replacing it, so * adding one marker cannot silently disable protection against all the others. @@ -1312,12 +1396,27 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { maxAgeDays: fileConfig.chatMessage?.maxAgeDays, injectOn: (fileConfig.chatMessage?.injectOn ?? DEFAULTS.chatMessage.injectOn) as "first" | "always", + recall: normalizeChatMessageRecall(fileConfig.chatMessage?.recall), + recallScope: normalizeChatMessageRecallScope(fileConfig.chatMessage?.recallScope), + similarityThreshold: + typeof fileConfig.chatMessage?.similarityThreshold === "number" && + Number.isFinite(fileConfig.chatMessage.similarityThreshold) + ? fileConfig.chatMessage.similarityThreshold + : DEFAULTS.chatMessage.similarityThreshold, filterInjectedPrompts: fileConfig.chatMessage?.filterInjectedPrompts ?? DEFAULTS.chatMessage.filterInjectedPrompts, injectionMarkers: normalizeInjectionMarkers(fileConfig.chatMessage?.injectionMarkers), captureChildSessions: fileConfig.chatMessage?.captureChildSessions ?? DEFAULTS.chatMessage.captureChildSessions, }, + dreaming: { + enabled: fileConfig.dreaming?.enabled ?? DEFAULTS.dreaming.enabled, + intervalHours: normalizeDreamingIntervalHours(fileConfig.dreaming?.intervalHours), + maxGroupsPerRun: normalizeDreamingMaxGroupsPerRun(fileConfig.dreaming?.maxGroupsPerRun), + trashRetentionDays: normalizeDreamingTrashRetentionDays( + fileConfig.dreaming?.trashRetentionDays + ), + }, }; } diff --git a/src/index.ts b/src/index.ts index 6a61ebb8..db72fc27 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { tool } from "@opencode-ai/plugin"; import { memoryClient } from "./services/client.js"; import { formatContextForPrompt } from "./services/context.js"; +import { shouldRefreshRecallOnPrompt } from "./services/memory-recall.js"; import { buildMemoryProvenanceSuffix } from "./services/memory-provenance.js"; import { getTags } from "./services/tags.js"; import { stripPrivateContent, isFullyPrivate } from "./services/privacy.js"; @@ -704,26 +705,16 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { }; const loadMemoryContext = async (sessionID: string): Promise => { - const listResult = await memoryClient.listMemories( - tags.project.tag, - CONFIG.chatMessage.maxMemories, - "project", - { - excludeSessionId: CONFIG.chatMessage.excludeCurrentSession ? sessionID : undefined, - createdAfter: CONFIG.chatMessage.maxAgeDays - ? Date.now() - CONFIG.chatMessage.maxAgeDays * 86400000 - : undefined, - } - ); - - const memories = listResult.success ? listResult.memories : []; + const { loadRecallMemoriesForSession } = await import("./services/memory-recall.js"); + const queryText = await userPromptManager.getLatestPromptTextForSession(sessionID); + const memories = await loadRecallMemoriesForSession(tags.project.tag, sessionID, queryText); // Still format an empty project-memory payload so injectProfile can emit // on its own (e.g. chatMessage.maxMemories: 0). const projectMemories = { - results: memories.map((m: any) => ({ - similarity: 1.0, - memory: m.summary, + results: memories.map((m) => ({ + similarity: m.similarity, + memory: m.memory, id: m.id, createdAt: m.createdAt, projectName: m.projectName, @@ -742,7 +733,11 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { // Internal V2 bridge; V1 still injects synthetic parts through chat.message. memoryContext: { enabled: () => isConfigured() && CONFIG.chatMessage.enabled, - refreshOnPrompt: () => CONFIG.chatMessage.injectOn === "always", + refreshOnPrompt: () => + shouldRefreshRecallOnPrompt( + CONFIG.chatMessage.recall ?? "relevant", + CONFIG.chatMessage.injectOn ?? "first" + ), capturePrompt: async ( input: { sessionID: string }, output: { message: { id: string }; parts: Part[] } @@ -788,8 +783,13 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { const lastMessage = messages.length > 0 ? messages[messages.length - 1] : null; const isAfterCompaction = lastMessage?.info?.summary === true; + // Relevance/both re-inject every authored turn so the query matches the + // current message (issue #460). Legacy recent keeps injectOn: first. const shouldInject = - CONFIG.chatMessage.injectOn === "always" || + shouldRefreshRecallOnPrompt( + CONFIG.chatMessage.recall ?? "relevant", + CONFIG.chatMessage.injectOn ?? "first" + ) || !hasNonSyntheticUserMessages || (isAfterCompaction && messages.filter( @@ -1311,6 +1311,9 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { if (webServer?.isServerOwner()) { const { cleanupService } = await import("./services/cleanup-service.js"); if (await cleanupService.shouldRunCleanup()) await cleanupService.runCleanup(); + + const { dreamingService } = await import("./services/dreaming-service.js"); + await dreamingService.maybeRun(directory); } } catch (error) { log("Idle processing error", { error: String(error) }); diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 6ae1d8c1..05f70a43 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -20,7 +20,6 @@ import { type CollectDailyStatsOptions, } from "./stats-service.js"; import { - deleteMemoryMarkdownFile, getMarkdownStatus, isMarkdownBackend, reindexMarkdownVault, @@ -479,28 +478,20 @@ export async function handleDeleteMemory( const memory = await tursoVectorSearch.getMemoryById(db, id); if (memory) { const metadata = safeJSONParse(memory.metadata); - if (isMarkdownBackend()) { - const rel = - metadata && typeof metadata.markdownRelPath === "string" - ? metadata.markdownRelPath - : null; - deleteMemoryMarkdownFile(id, rel); - } + // Soft-delete keeps the row in trash (recoverable). Markdown files stay + // until hard-purge so restore can re-surface the same note. if (cascade) { const linkedPromptId = metadata?.promptId; if (linkedPromptId) await userPromptManager.deletePrompt(linkedPromptId); } - await tursoVectorSearch.deleteVector(db, id); - await tursoShardManager.decrementVectorCount(shard.id); + const ok = await tursoVectorSearch.softDeleteVector(db, id); + if (!ok) return { success: false, error: "Memory not found" }; return { success: true, data: { deletedPrompt: cascade && !!metadata?.promptId }, }; } } - if (isMarkdownBackend()) { - deleteMemoryMarkdownFile(id); - } return { success: false, error: "Memory not found" }; } catch (error) { log("handleDeleteMemory: error", { error: String(error) }); @@ -508,6 +499,76 @@ export async function handleDeleteMemory( } } +export async function handleListTrash(limit = 200): Promise< + ApiResponse< + Array<{ + id: string; + content: string; + deletedAt: number; + mergedInto?: string; + projectName?: string; + containerTag: string; + }> + > +> { + try { + const shards = await getAllMemoryShards(); + const items: Array<{ + id: string; + content: string; + deletedAt: number; + mergedInto?: string; + projectName?: string; + containerTag: string; + }> = []; + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const rows = await tursoVectorSearch.listDeletedMemories(db, limit); + for (const row of rows) { + const meta = safeJSONParse(row.metadata) || {}; + items.push({ + id: String(row.id), + content: String(row.content), + deletedAt: Number(row.deleted_at), + mergedInto: typeof meta.mergedInto === "string" ? meta.mergedInto : undefined, + projectName: row.project_name ? String(row.project_name) : undefined, + containerTag: String(row.container_tag), + }); + } + } + items.sort((a, b) => b.deletedAt - a.deletedAt); + return { success: true, data: items.slice(0, limit) }; + } catch (error) { + log("handleListTrash: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + +export async function handleRestoreMemory(id: string): Promise> { + try { + if (!id) return { success: false, error: "id is required" }; + const shards = await getAllMemoryShards(); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id, { includeDeleted: true }); + if (!memory || memory.deleted_at == null) continue; + const ok = await tursoVectorSearch.restoreVector(db, id); + if (!ok) return { success: false, error: "Failed to restore memory" }; + // Clear merge pointer so the restored row stands alone again. + const meta = safeJSONParse(memory.metadata) || {}; + if (meta.mergedInto) { + delete meta.mergedInto; + await db.run(`UPDATE memories SET metadata = ? WHERE id = ?`, [JSON.stringify(meta), id]); + } + return { success: true, data: { restored: true } }; + } + return { success: false, error: "Memory not found in trash" }; + } catch (error) { + log("handleRestoreMemory: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + export async function handleBulkDelete( ids: string[], cascade: boolean = false @@ -585,14 +646,21 @@ export async function handleUpdateMemory( const metadataJson = JSON.stringify(existingMeta); + const isPinned = Number(existingMemory.is_pinned ?? 0) === 1 ? 1 : 0; + const deletedAt = + existingMemory.deleted_at == null || existingMemory.deleted_at === undefined + ? null + : Number(existingMemory.deleted_at); + await db.transaction("write", async (tx) => { await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [id] }); await tx.execute({ sql: ` INSERT INTO memories ( id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, - metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url - ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url, + is_pinned, deleted_at + ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, args: [ id, @@ -616,6 +684,8 @@ export async function handleUpdateMemory( existingMemory.project_path ? String(existingMemory.project_path) : null, existingMemory.project_name ? String(existingMemory.project_name) : null, existingMemory.git_repo_url ? String(existingMemory.git_repo_url) : null, + isPinned, + deletedAt, ], }); }); @@ -1049,6 +1119,41 @@ export async function handleRunDeduplication(): Promise< } } +export async function handleDreamingStatus(): Promise< + ApiResponse> +> { + try { + const { dreamingService } = await import("./dreaming-service.js"); + return { success: true, data: dreamingService.getStatus() }; + } catch (error) { + log("handleDreamingStatus: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + +export async function handleRunDreaming( + directory?: string +): Promise< + ApiResponse< + Awaited> + > +> { + try { + if (!CONFIG.dreaming.enabled) { + return { + success: false, + error: "Dreaming is disabled. Set dreaming.enabled to true in opencode-mem.jsonc.", + }; + } + const { dreamingService } = await import("./dreaming-service.js"); + const result = await dreamingService.run(directory); + return { success: true, data: result }; + } catch (error) { + log("handleRunDreaming: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + export async function handleDetectMigration(): Promise< ApiResponse<{ needsMigration: boolean; @@ -1647,7 +1752,7 @@ export async function handleDetectTagMigration(): Promise< for (const shard of projectShards) { const db = await tursoConnectionManager.getConnection(shard.dbPath); const row = await db.get( - "SELECT COUNT(*) as count FROM memories WHERE tags IS NULL OR tags = ''" + "SELECT COUNT(*) as count FROM memories WHERE (deleted_at IS NULL) AND (tags IS NULL OR tags = '')" ); untaggedCount += Number(row?.count ?? 0); } @@ -1864,7 +1969,7 @@ export async function handleRunTagMigrationBatch( for (const shard of projectShards) { const db = await tursoConnectionManager.getConnection(shard.dbPath); - const memories = await db.all("SELECT * FROM memories"); + const memories = await db.all("SELECT * FROM memories WHERE deleted_at IS NULL"); for (const m of memories) { allMemories.push({ memory: m, shard }); } diff --git a/src/services/cleanup-service.ts b/src/services/cleanup-service.ts index 234312d4..d2b79cc9 100644 --- a/src/services/cleanup-service.ts +++ b/src/services/cleanup-service.ts @@ -69,10 +69,12 @@ export class CleanupService { for (const shard of allShards) { const db = await tursoConnectionManager.getConnection(shard.dbPath); + // Ensure deleted_at (v3) exists before filtering on it. + await tursoShardManager.ensureShardSchema(db); const oldMemories = await db.all( ` SELECT id, container_tag, is_pinned FROM memories - WHERE updated_at < ? + WHERE updated_at < ? AND deleted_at IS NULL `, [cutoffTime] ); @@ -101,6 +103,21 @@ export class CleanupService { log("Cleanup: delete error", { memoryId: memory.id, error: String(error) }); } } + + // Hard-purge soft-deleted trash past retention. + const trashCutoff = Date.now() - CONFIG.dreaming.trashRetentionDays * 24 * 60 * 60 * 1000; + try { + const purged = await tursoVectorSearch.purgeDeletedBefore(db, trashCutoff); + for (const _id of purged) { + await tursoShardManager.decrementVectorCount(shard.id); + totalDeleted++; + } + if (purged.length > 0) { + log("Cleanup: purged trash", { count: purged.length, shardId: shard.id }); + } + } catch (error) { + log("Cleanup: trash purge error", { shardId: shard.id, error: String(error) }); + } } const promptsDeleted = promptCleanupResult.deleted - linkedMemoryIds.size; diff --git a/src/services/client.ts b/src/services/client.ts index feb67807..12a00d1a 100644 --- a/src/services/client.ts +++ b/src/services/client.ts @@ -13,11 +13,7 @@ import { CONFIG } from "../config.js"; import { log, serializeError } from "./logger.js"; import type { MemoryType } from "../types/index.js"; import type { MemoryListFilters, MemoryRecord } from "./turso/types.js"; -import { - deleteMemoryMarkdownFile, - isMarkdownBackend, - writeMemoryMarkdownFile, -} from "./markdown/backend.js"; +import { isMarkdownBackend, writeMemoryMarkdownFile } from "./markdown/backend.js"; import { pluginMemoryRelPath } from "./markdown/vault.js"; export type MemoryScope = "project" | "all-projects"; @@ -123,7 +119,12 @@ export class LocalMemoryClient { } } - async searchMemories(query: string, containerTag: string, scope: MemoryScope = "project") { + async searchMemories( + query: string, + containerTag: string, + scope: MemoryScope = "project", + options: { limit?: number; similarityThreshold?: number } = {} + ) { try { await this.initialize(); @@ -139,12 +140,15 @@ export class LocalMemoryClient { return { success: true as const, results: [], total: 0, timing: 0 }; } + const limit = options.limit ?? CONFIG.maxMemories; + const similarityThreshold = options.similarityThreshold ?? CONFIG.similarityThreshold; + const { results, warnings } = await tursoVectorSearch.searchAcrossShards( shards, queryVector, scope === "all-projects" ? "" : containerTag, - CONFIG.maxMemories, - CONFIG.similarityThreshold, + limit, + similarityThreshold, query ); @@ -288,22 +292,12 @@ export class LocalMemoryClient { const memory = await tursoVectorSearch.getMemoryById(db, memoryId); if (memory) { - if (isMarkdownBackend()) { - const meta = safeJSONParse(memory.metadata); - const rel = - meta && typeof meta.markdownRelPath === "string" ? meta.markdownRelPath : null; - deleteMemoryMarkdownFile(memoryId, rel); - } - await tursoVectorSearch.deleteVector(db, memoryId); - await tursoShardManager.decrementVectorCount(shard.id); - return { success: true }; + // Soft-delete into trash (recoverable). Hard-purge happens via cleanup. + const ok = await tursoVectorSearch.softDeleteVector(db, memoryId); + return ok ? { success: true } : { success: false, error: "Memory not found" }; } } - if (isMarkdownBackend()) { - deleteMemoryMarkdownFile(memoryId); - } - return { success: false, error: "Memory not found" }; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); diff --git a/src/services/dreaming-lock.ts b/src/services/dreaming-lock.ts new file mode 100644 index 00000000..4efc75cd --- /dev/null +++ b/src/services/dreaming-lock.ts @@ -0,0 +1,87 @@ +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { CONFIG } from "../config.js"; +import { log } from "./logger.js"; + +const LOCK_FILE = "dreaming.lock"; +/** Treat locks older than this as stale (crashed process). */ +const STALE_MS = 2 * 60 * 60 * 1000; + +interface LockPayload { + pid: number; + acquiredAt: number; +} + +function lockPath(): string { + return join(CONFIG.storagePath, LOCK_FILE); +} + +function readLock(): LockPayload | null { + try { + const raw = JSON.parse(readFileSync(lockPath(), "utf-8")) as LockPayload; + if (typeof raw.pid !== "number" || typeof raw.acquiredAt !== "number") return null; + return raw; + } catch { + return null; + } +} + +function isProcessAlive(pid: number): boolean { + if (!Number.isFinite(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** + * Exclusive cross-process lock for dreaming runs. + * Returns a release function, or null if another live owner holds the lock. + */ +export function tryAcquireDreamingLock(): (() => void) | null { + const dir = CONFIG.storagePath; + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + const path = lockPath(); + const payload: LockPayload = { pid: process.pid, acquiredAt: Date.now() }; + + const existing = existsSync(path) ? readLock() : null; + if (existing) { + const stale = Date.now() - existing.acquiredAt > STALE_MS; + const alive = isProcessAlive(existing.pid); + // Same pid still counts as held (in-process re-entry must fail too). + if (!stale && alive) { + log("dreaming lock: held by another owner", { + pid: existing.pid, + acquiredAt: existing.acquiredAt, + }); + return null; + } + try { + unlinkSync(path); + } catch { + // race: another process may have removed it + } + } + + try { + // wx: fail if recreated between unlink and create + writeFileSync(path, JSON.stringify(payload), { flag: "wx" }); + } catch { + log("dreaming lock: lost create race"); + return null; + } + + let released = false; + return () => { + if (released) return; + released = true; + try { + const current = readLock(); + if (current?.pid === process.pid) unlinkSync(path); + } catch { + // ignore + } + }; +} diff --git a/src/services/dreaming-service.ts b/src/services/dreaming-service.ts new file mode 100644 index 00000000..04b6e4e2 --- /dev/null +++ b/src/services/dreaming-service.ts @@ -0,0 +1,395 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import { CONFIG } from "../config.js"; +import { embeddingService } from "./embedding.js"; +import { log } from "./logger.js"; +import { cosineSimilarity } from "../utils/math.js"; +import { ensureTursoReady } from "./turso/ready.js"; +import { tursoConnectionManager } from "./turso/connection-manager.js"; +import { tursoShardManager } from "./turso/shard-manager.js"; +import { tursoVectorSearch } from "./turso/vector-search.js"; +import { parseExtractedVector, formatTagsForEmbedding } from "./turso/vector-utils.js"; +import { getV2Client } from "./ai/opencode-provider.js"; +import { tryAcquireDreamingLock } from "./dreaming-lock.js"; + +export interface DreamingNearDupGroup { + representative: { + id: string; + content: string; + containerTag: string; + createdAt: number; + isPinned: boolean; + }; + duplicates: Array<{ + id: string; + content: string; + similarity: number; + isPinned: boolean; + }>; +} + +export interface DreamingRunResult { + groupsConsidered: number; + groupsMerged: number; + groupsSkipped: number; + memoriesSoftDeleted: number; +} + +interface DreamingState { + lastRunAt: number; +} + +const mergeSchema = z.object({ + action: z.enum(["merge", "skip"]), + content: z.string().optional(), + keepId: z.string().optional(), + dropIds: z.array(z.string()).optional(), + reason: z.string().optional(), +}); + +function statePath(): string { + return join(CONFIG.storagePath, "dreaming-state.json"); +} + +function readState(): DreamingState { + try { + const path = statePath(); + if (!existsSync(path)) return { lastRunAt: 0 }; + const raw = JSON.parse(readFileSync(path, "utf-8")) as DreamingState; + return { lastRunAt: Number(raw.lastRunAt) || 0 }; + } catch { + return { lastRunAt: 0 }; + } +} + +function writeState(state: DreamingState): void { + const dir = CONFIG.storagePath; + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(statePath(), JSON.stringify(state), "utf-8"); +} + +export async function findNearDuplicateGroupsForDreaming(): Promise { + await ensureTursoReady(); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); + const allShards = [...userShards, ...projectShards]; + const groups: DreamingNearDupGroup[] = []; + const threshold = CONFIG.deduplicationSimilarityThreshold; + + for (const shard of allShards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.getAllMemoriesWithExtractedVectors(db); + const processedIds = new Set(); + + for (let i = 0; i < memories.length; i++) { + const mem1 = memories[i]!; + if (Number(mem1.is_pinned ?? 0) === 1) continue; + const vector1 = parseExtractedVector(mem1.vector_json); + if (!vector1 || processedIds.has(String(mem1.id))) continue; + + const similarGroup: DreamingNearDupGroup = { + representative: { + id: String(mem1.id), + content: String(mem1.content), + containerTag: String(mem1.container_tag), + createdAt: Number(mem1.created_at), + isPinned: false, + }, + duplicates: [], + }; + + for (let j = i + 1; j < memories.length; j++) { + const mem2 = memories[j]!; + if (Number(mem2.is_pinned ?? 0) === 1) continue; + const vector2 = parseExtractedVector(mem2.vector_json); + if (!vector2 || processedIds.has(String(mem2.id))) continue; + if (mem1.container_tag !== mem2.container_tag) continue; + + const similarity = cosineSimilarity(vector1, vector2); + if (similarity >= threshold && similarity < 1.0) { + similarGroup.duplicates.push({ + id: String(mem2.id), + content: String(mem2.content), + similarity, + isPinned: false, + }); + processedIds.add(String(mem2.id)); + } + } + + if (similarGroup.duplicates.length > 0) { + processedIds.add(String(mem1.id)); + groups.push(similarGroup); + } + } + } + + return groups; +} + +function buildMergePrompt(group: DreamingNearDupGroup): string { + const items = [ + { id: group.representative.id, content: group.representative.content }, + ...group.duplicates.map((d) => ({ id: d.id, content: d.content })), + ]; + return `You consolidate near-duplicate project memories into one clearer entry. + +Rules: +- Prefer action "merge" only when the memories clearly describe the same fact/fix/gotcha. +- The merged content MUST retain important details, including why something failed when present. +- Do NOT invent facts. Do NOT drop unique constraints, commands, paths, or error messages. +- keepId must be one of the input ids (prefer the most complete/recent). +- dropIds must list every other id when merging. +- Use action "skip" when memories are related but not safe to merge. + +Return JSON matching the schema. + +Memories: +${JSON.stringify(items, null, 2)}`; +} + +export class DreamingService { + private isRunning = false; + + getStatus() { + const state = readState(); + return { + enabled: CONFIG.dreaming.enabled, + intervalHours: CONFIG.dreaming.intervalHours, + maxGroupsPerRun: CONFIG.dreaming.maxGroupsPerRun, + trashRetentionDays: CONFIG.dreaming.trashRetentionDays, + isRunning: this.isRunning, + lastRunAt: state.lastRunAt || null, + }; + } + + async shouldRun(): Promise { + if (!CONFIG.dreaming.enabled) return false; + if (this.isRunning) return false; + if (!CONFIG.autoCaptureProviderStatus?.ready) return false; + const state = readState(); + const intervalMs = CONFIG.dreaming.intervalHours * 60 * 60 * 1000; + return Date.now() - state.lastRunAt >= intervalMs; + } + + async maybeRun(directory?: string): Promise { + if (!(await this.shouldRun())) return null; + try { + return await this.run(directory); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes("another process") || message.includes("already running")) { + log("dreaming: skipped due to lock contention", { error: message }); + return null; + } + throw error; + } + } + + /** Manual or gated run. Does not re-check interval; caller uses maybeRun for that. */ + async run(directory?: string): Promise { + if (this.isRunning) { + throw new Error("Dreaming already running"); + } + const releaseLock = tryAcquireDreamingLock(); + if (!releaseLock) { + throw new Error("Dreaming already running in another process"); + } + this.isRunning = true; + const result: DreamingRunResult = { + groupsConsidered: 0, + groupsMerged: 0, + groupsSkipped: 0, + memoriesSoftDeleted: 0, + }; + + try { + await ensureTursoReady(); + const groups = await findNearDuplicateGroupsForDreaming(); + const batch = groups.slice(0, CONFIG.dreaming.maxGroupsPerRun); + result.groupsConsidered = batch.length; + + if (batch.length === 0) { + writeState({ lastRunAt: Date.now() }); + return result; + } + + const v2Client = getV2Client(); + if (!v2Client || !CONFIG.opencodeProvider || !CONFIG.opencodeModel) { + log("dreaming: skipped (no opencode structured-output client)"); + return result; + } + + const { generateStructuredOutput } = await import("./ai/opencode-provider.js"); + + for (const group of batch) { + try { + const decision = await generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + ...(CONFIG.opencodeVariant ? { variant: CONFIG.opencodeVariant } : {}), + systemPrompt: + "You are a careful memory consolidation assistant. Return only structured JSON.", + userPrompt: buildMergePrompt(group), + schema: mergeSchema, + directory, + }); + + if (decision.action !== "merge" || !decision.content?.trim()) { + result.groupsSkipped++; + continue; + } + + const memberIds = new Set([ + group.representative.id, + ...group.duplicates.map((d) => d.id), + ]); + const keepId = + decision.keepId && memberIds.has(decision.keepId) + ? decision.keepId + : group.representative.id; + const dropIds = (decision.dropIds || []) + .filter((id) => memberIds.has(id) && id !== keepId) + .filter((id, idx, arr) => arr.indexOf(id) === idx); + const finalDrop = + dropIds.length > 0 ? dropIds : [...memberIds].filter((id) => id !== keepId); + + if (finalDrop.length === 0) { + result.groupsSkipped++; + continue; + } + + const applied = await this.applyMerge( + keepId, + decision.content.trim(), + finalDrop, + group.representative.containerTag + ); + if (applied) { + result.groupsMerged++; + result.memoriesSoftDeleted += finalDrop.length; + } else { + result.groupsSkipped++; + } + } catch (error) { + result.groupsSkipped++; + log("dreaming: group failed", { error: String(error) }); + } + } + + writeState({ lastRunAt: Date.now() }); + log("dreaming: run complete", result); + return result; + } finally { + this.isRunning = false; + releaseLock(); + } + } + + private async applyMerge( + keepId: string, + content: string, + dropIds: string[], + containerTag: string + ): Promise { + const shards = [ + ...(await tursoShardManager.getAllShards("user", "")), + ...(await tursoShardManager.getAllShards("project", "")), + ]; + + let keepShard = null; + let keepRow: Record | null = null; + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const row = await tursoVectorSearch.getMemoryById(db, keepId); + if (row) { + keepShard = shard; + keepRow = row; + break; + } + } + if (!keepShard || !keepRow) return false; + if (Number(keepRow.is_pinned ?? 0) === 1) return false; + + await embeddingService.warmup(); + const vector = await embeddingService.embedWithTimeout(content, { task: "document" }); + const existingTags = keepRow.tags ? String(keepRow.tags) : ""; + const tags = existingTags + ? existingTags + .split(",") + .map((t) => t.trim()) + .filter(Boolean) + : []; + let tagsVector: Float32Array | undefined; + if (tags.length > 0) { + tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags), { + task: "document", + }); + } + + const meta = + keepRow.metadata && typeof keepRow.metadata === "string" + ? (JSON.parse(keepRow.metadata) as Record) + : {}; + const mergedFrom = Array.isArray(meta.mergedFrom) ? [...(meta.mergedFrom as string[])] : []; + for (const id of dropIds) { + if (!mergedFrom.includes(id)) mergedFrom.push(id); + } + meta.mergedFrom = mergedFrom; + meta.source = typeof meta.source === "string" ? meta.source : "dreaming"; + + const db = await tursoConnectionManager.getConnection(keepShard.dbPath); + const updatedAt = Date.now(); + const isPinned = Number(keepRow.is_pinned ?? 0) === 1 ? 1 : 0; + + await db.transaction("write", async (tx) => { + await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [keepId] }); + await tx.execute({ + sql: ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url, + is_pinned, deleted_at + ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL) + `, + args: [ + keepId, + content, + JSON.stringify(Array.from(vector)), + ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), + containerTag || String(keepRow.container_tag), + tags.length > 0 ? tags.join(",") : null, + keepRow.type ? String(keepRow.type) : null, + Number(keepRow.created_at), + updatedAt, + JSON.stringify(meta), + keepRow.session_id ? String(keepRow.session_id) : null, + keepRow.display_name ? String(keepRow.display_name) : null, + keepRow.user_name ? String(keepRow.user_name) : null, + keepRow.user_email ? String(keepRow.user_email) : null, + keepRow.project_path ? String(keepRow.project_path) : null, + keepRow.project_name ? String(keepRow.project_name) : null, + keepRow.git_repo_url ? String(keepRow.git_repo_url) : null, + isPinned, + ], + }); + }); + + for (const dropId of dropIds) { + for (const shard of shards) { + const dropDb = await tursoConnectionManager.getConnection(shard.dbPath); + const dropRow = await tursoVectorSearch.getMemoryById(dropDb, dropId); + if (!dropRow) continue; + if (Number(dropRow.is_pinned ?? 0) === 1) continue; + await tursoVectorSearch.softDeleteVector(dropDb, dropId, { mergedInto: keepId }); + break; + } + } + + return true; + } +} + +export const dreamingService = new DreamingService(); diff --git a/src/services/memory-recall.ts b/src/services/memory-recall.ts new file mode 100644 index 00000000..7cf54b58 --- /dev/null +++ b/src/services/memory-recall.ts @@ -0,0 +1,244 @@ +import { CONFIG } from "../config.js"; +import { memoryClient, type MemoryScope } from "./client.js"; +import { log } from "./logger.js"; +import type { SearchResult } from "./turso/types.js"; + +export type RecallMode = "recent" | "relevant" | "both"; +export type RecallScopeSetting = "project" | "all-projects" | "default"; + +export interface RecallMemory { + id: string; + memory: string; + similarity: number; + createdAt?: number | string; + projectName?: string; + containerTag?: string; + metadata?: Record; + sessionId?: string; +} + +export const PROJECT_BOOST = 0.05; + +/** + * Relevance-based recall must re-run against each new user message (issue #460). + * Legacy `"recent"` listing keeps the historical injectOn gate (`first` vs `always`). + */ +export function shouldRefreshRecallOnPrompt( + recall: RecallMode, + injectOn: "first" | "always" +): boolean { + if (injectOn === "always") return true; + return recall === "relevant" || recall === "both"; +} + +export function resolveRecallScope( + recallScope: RecallScopeSetting, + defaultScope: "project" | "all-projects" +): MemoryScope { + if (recallScope === "default") return defaultScope; + return recallScope; +} + +export function applyProjectBoost( + results: RecallMemory[], + currentContainerTag: string, + boost = PROJECT_BOOST +): RecallMemory[] { + return results + .map((r) => ({ + ...r, + similarity: Math.min( + 1, + r.similarity + (r.containerTag && r.containerTag === currentContainerTag ? boost : 0) + ), + })) + .sort((a, b) => b.similarity - a.similarity); +} + +export function filterRecallResults( + results: RecallMemory[], + filters: { excludeSessionId?: string; createdAfter?: number } +): RecallMemory[] { + return results.filter((r) => { + if (filters.excludeSessionId) { + const sid = + r.sessionId ?? + (typeof r.metadata?.sessionID === "string" ? r.metadata.sessionID : undefined); + if (sid === filters.excludeSessionId) return false; + } + if (filters.createdAfter !== undefined) { + const t = typeof r.createdAt === "string" ? Date.parse(r.createdAt) : Number(r.createdAt); + if (Number.isFinite(t) && t < filters.createdAfter) return false; + } + return true; + }); +} + +export function mergeBoth( + relevant: RecallMemory[], + recent: RecallMemory[], + maxMemories: number +): RecallMemory[] { + const seen = new Set(); + const out: RecallMemory[] = []; + for (const r of relevant) { + if (out.length >= maxMemories) break; + if (seen.has(r.id)) continue; + seen.add(r.id); + out.push(r); + } + for (const r of recent) { + if (out.length >= maxMemories) break; + if (seen.has(r.id)) continue; + seen.add(r.id); + out.push(r); + } + return out; +} + +function searchResultToRecall(r: SearchResult): RecallMemory { + const meta = r.metadata; + const sessionId = + typeof (r as { sessionId?: string }).sessionId === "string" + ? (r as { sessionId?: string }).sessionId + : typeof meta?.sessionID === "string" + ? meta.sessionID + : undefined; + return { + id: r.id, + memory: r.memory, + similarity: r.similarity, + createdAt: r.createdAt, + projectName: r.projectName, + containerTag: (r as { containerTag?: string }).containerTag, + metadata: meta, + sessionId, + }; +} + +function listRowToRecall(m: { + id: string; + summary: string; + createdAt?: string; + projectName?: string; + metadata?: Record; +}): RecallMemory { + const sessionId = typeof m.metadata?.sessionID === "string" ? m.metadata.sessionID : undefined; + return { + id: m.id, + memory: m.summary, + similarity: 1.0, + createdAt: m.createdAt, + projectName: m.projectName, + metadata: m.metadata, + sessionId, + }; +} + +export interface LoadRecallOptions { + containerTag: string; + sessionID: string; + queryText: string | null; + maxMemories: number; + recall: RecallMode; + recallScope: RecallScopeSetting; + defaultScope: "project" | "all-projects"; + excludeCurrentSession: boolean; + maxAgeDays?: number; + similarityThreshold: number; +} + +/** + * Select memories for chat auto-injection according to chatMessage.recall. + * Falls back to recent listing when relevant/both is requested but no query text + * is available (e.g. V2 context rehydrate without a fresh prompt). + */ +export async function loadRecallMemories(opts: LoadRecallOptions): Promise { + const scope = resolveRecallScope(opts.recallScope, opts.defaultScope); + const filters = { + excludeSessionId: opts.excludeCurrentSession ? opts.sessionID : undefined, + createdAfter: opts.maxAgeDays ? Date.now() - opts.maxAgeDays * 86400000 : undefined, + }; + const listFilters = { + excludeSessionId: filters.excludeSessionId, + createdAfter: filters.createdAfter, + }; + + let mode = opts.recall; + if ((mode === "relevant" || mode === "both") && !opts.queryText?.trim()) { + log("memory recall: no query text, falling back to recent", { + sessionID: opts.sessionID, + requested: mode, + }); + mode = "recent"; + } + + const loadRecent = async (): Promise => { + const listResult = await memoryClient.listMemories( + opts.containerTag, + opts.maxMemories, + scope, + listFilters + ); + if (!listResult.success) return []; + return listResult.memories.map(listRowToRecall); + }; + + const loadRelevant = async (): Promise => { + const query = opts.queryText!.trim(); + // Over-fetch so session/age filters and project boost can still fill the slot budget. + const fetchLimit = Math.max(opts.maxMemories * 3, opts.maxMemories); + const searchRes = await memoryClient.searchMemories(query, opts.containerTag, scope, { + limit: fetchLimit, + similarityThreshold: opts.similarityThreshold, + }); + if (!searchRes.success) return []; + let results = searchRes.results.map(searchResultToRecall); + results = filterRecallResults(results, filters); + if (scope === "all-projects") { + results = applyProjectBoost(results, opts.containerTag); + } + return results.slice(0, opts.maxMemories); + }; + + if (opts.maxMemories <= 0) return []; + + if (mode === "recent") { + return loadRecent(); + } + + if (mode === "relevant") { + return loadRelevant(); + } + + // both + const relevant = await loadRelevant(); + if (relevant.length >= opts.maxMemories) return relevant.slice(0, opts.maxMemories); + const recent = await loadRecent(); + return mergeBoth(relevant, recent, opts.maxMemories); +} + +/** Convenience wrapper using CONFIG + session prompt text. */ +export async function loadRecallMemoriesForSession( + containerTag: string, + sessionID: string, + queryText: string | null +): Promise { + const threshold = + typeof CONFIG.chatMessage.similarityThreshold === "number" + ? CONFIG.chatMessage.similarityThreshold + : CONFIG.similarityThreshold; + + return loadRecallMemories({ + containerTag, + sessionID, + queryText, + maxMemories: CONFIG.chatMessage.maxMemories ?? 3, + recall: CONFIG.chatMessage.recall ?? "relevant", + recallScope: CONFIG.chatMessage.recallScope ?? "default", + defaultScope: CONFIG.memory.defaultScope ?? "project", + excludeCurrentSession: CONFIG.chatMessage.excludeCurrentSession ?? true, + maxAgeDays: CONFIG.chatMessage.maxAgeDays, + similarityThreshold: threshold, + }); +} diff --git a/src/services/stats-service.ts b/src/services/stats-service.ts index e06b1824..ddcb1f55 100644 --- a/src/services/stats-service.ts +++ b/src/services/stats-service.ts @@ -290,7 +290,9 @@ export async function collectShardStats(shards: ShardInfo[]): Promise("SELECT container_tag, type, COUNT(*) c FROM memories GROUP BY container_tag, type"); + }>( + "SELECT container_tag, type, COUNT(*) c FROM memories WHERE deleted_at IS NULL GROUP BY container_tag, type" + ); for (const row of scopeTypeCounts) { const containerTag = String(row.container_tag ?? ""); const type = row.type ? String(row.type) : ""; @@ -301,14 +303,14 @@ export async function collectShardStats(shards: ShardInfo[]): Promise( - "SELECT COUNT(*) c, MIN(created_at) mn, MAX(created_at) mx FROM memories" + "SELECT COUNT(*) c, MIN(created_at) mn, MAX(created_at) mx FROM memories WHERE deleted_at IS NULL" ); stats.memoryCount = Number(aggregate?.c ?? 0); stats.oldestCreatedAt = toISOStringOrNull(aggregate?.mn); stats.newestCreatedAt = toISOStringOrNull(aggregate?.mx); const duplicateTags = await db.all<{ c: number | bigint }>( - "SELECT tags, COUNT(*) c FROM memories WHERE tags IS NOT NULL AND tags != '' GROUP BY tags HAVING COUNT(*) > 1" + "SELECT tags, COUNT(*) c FROM memories WHERE deleted_at IS NULL AND tags IS NOT NULL AND tags != '' GROUP BY tags HAVING COUNT(*) > 1" ); stats.duplicateTagCount = duplicateTags.reduce( (total, row) => total + Math.max(0, Number(row.c) - 1), diff --git a/src/services/turso/schema-migrations.ts b/src/services/turso/schema-migrations.ts index ceeb270f..ff80af6b 100644 --- a/src/services/turso/schema-migrations.ts +++ b/src/services/turso/schema-migrations.ts @@ -139,6 +139,14 @@ export function memoryShardMigrations(dimensions: number): SchemaMigration[] { }, ], }, + { + version: 3, + description: "Soft-delete / trash via deleted_at for recoverable memory removal", + statements: [ + { sql: `ALTER TABLE memories ADD COLUMN deleted_at INTEGER` }, + { sql: `CREATE INDEX IF NOT EXISTS idx_deleted_at ON memories(deleted_at)` }, + ], + }, ]; } diff --git a/src/services/turso/types.ts b/src/services/turso/types.ts index 884f7ca7..120fef32 100644 --- a/src/services/turso/types.ts +++ b/src/services/turso/types.ts @@ -42,10 +42,13 @@ export interface SearchResult { createdAt?: number; tags?: string[]; metadata?: Record; + containerTag?: string; + sessionId?: string; displayName?: string; userName?: string; userEmail?: string; projectPath?: string; projectName?: string; gitRepoUrl?: string; + isPinned?: unknown; } diff --git a/src/services/turso/vector-search.ts b/src/services/turso/vector-search.ts index 63295ccc..f6249d2e 100644 --- a/src/services/turso/vector-search.ts +++ b/src/services/turso/vector-search.ts @@ -11,6 +11,10 @@ import { } from "./vector-utils.js"; import type { TursoDb, TursoTx } from "./turso-db.js"; +/** Active (non-trashed) rows. Legacy DBs without deleted_at treat NULL as active. */ +export const ACTIVE_MEMORY_SQL = `(deleted_at IS NULL)`; +export const ACTIVE_MEMORY_ALIAS_SQL = `(m.deleted_at IS NULL)`; + function parseMetadata(value: unknown): Record | undefined { if (!value || typeof value !== "string") return undefined; try { @@ -134,7 +138,7 @@ export class TursoVectorSearch { THEN vector_distance_cos(tags_vector, vector32(?)) ELSE NULL END AS tags_dist FROM memories - WHERE id IN (${placeholders}) + WHERE id IN (${placeholders}) AND ${ACTIVE_MEMORY_SQL} ` : ` SELECT id, content, tags, created_at, metadata, container_tag, @@ -145,7 +149,7 @@ export class TursoVectorSearch { THEN vector_distance_cos(tags_vector, vector32(?)) ELSE NULL END AS tags_dist FROM memories - WHERE id IN (${placeholders}) AND container_tag = ? + WHERE id IN (${placeholders}) AND container_tag = ? AND ${ACTIVE_MEMORY_SQL} `, containerTag === "" ? [queryJson, queryJson, ...ids] @@ -191,6 +195,7 @@ export class TursoVectorSearch { tags: memoryTagsStr ? memoryTagsStr.split(",") : [], metadata: parseMetadata(row.metadata), containerTag: String(row.container_tag), + sessionId: row.session_id ? String(row.session_id) : undefined, displayName: row.display_name ? String(row.display_name) : undefined, userName: row.user_name ? String(row.user_name) : undefined, userEmail: row.user_email ? String(row.user_email) : undefined, @@ -226,12 +231,14 @@ export class TursoVectorSearch { containerTag === "" ? ` SELECT id FROM memories - WHERE content LIKE ? ESCAPE '\\' OR IFNULL(tags, '') LIKE ? ESCAPE '\\' + WHERE ${ACTIVE_MEMORY_SQL} + AND (content LIKE ? ESCAPE '\\' OR IFNULL(tags, '') LIKE ? ESCAPE '\\') LIMIT ? ` : ` SELECT id FROM memories WHERE container_tag = ? + AND ${ACTIVE_MEMORY_SQL} AND (content LIKE ? ESCAPE '\\' OR IFNULL(tags, '') LIKE ? ESCAPE '\\') LIMIT ? `, @@ -257,7 +264,7 @@ export class TursoVectorSearch { ? ` SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist FROM memories m - WHERE m.${columnName} IS NOT NULL + WHERE m.${columnName} IS NOT NULL AND ${ACTIVE_MEMORY_ALIAS_SQL} ORDER BY dist ASC LIMIT ? ` @@ -265,6 +272,7 @@ export class TursoVectorSearch { SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist FROM memories m WHERE m.${columnName} IS NOT NULL AND m.container_tag = ? + AND ${ACTIVE_MEMORY_ALIAS_SQL} ORDER BY dist ASC LIMIT ? `, @@ -336,6 +344,58 @@ export class TursoVectorSearch { await db.run(`DELETE FROM memories WHERE id = ?`, [memoryId]); } + async softDeleteVector( + db: TursoDb, + memoryId: string, + metaPatch?: Record + ): Promise { + await this.prepareShardDb(db); + const existing = await this.getMemoryById(db, memoryId, { includeDeleted: false }); + if (!existing) return false; + const meta = parseMetadata(existing.metadata) || {}; + if (metaPatch) Object.assign(meta, metaPatch); + const changes = await db.run( + `UPDATE memories SET deleted_at = ?, metadata = ?, updated_at = ? WHERE id = ? AND ${ACTIVE_MEMORY_SQL}`, + [Date.now(), JSON.stringify(meta), Date.now(), memoryId] + ); + return changes > 0; + } + + async restoreVector(db: TursoDb, memoryId: string): Promise { + await this.prepareShardDb(db); + const changes = await db.run( + `UPDATE memories SET deleted_at = NULL, updated_at = ? WHERE id = ? AND deleted_at IS NOT NULL`, + [Date.now(), memoryId] + ); + return changes > 0; + } + + async listDeletedMemories(db: TursoDb, limit = 200): Promise[]> { + await this.prepareShardDb(db); + return db.all( + ` + SELECT * FROM memories + WHERE deleted_at IS NOT NULL + ORDER BY deleted_at DESC + LIMIT ? + `, + [limit] + ); + } + + async purgeDeletedBefore(db: TursoDb, cutoffMs: number): Promise { + await this.prepareShardDb(db); + const rows = await db.all( + `SELECT id FROM memories WHERE deleted_at IS NOT NULL AND deleted_at < ?`, + [cutoffMs] + ); + const ids = rows.map((r) => String(r.id)); + for (const id of ids) { + await this.deleteVector(db, id); + } + return ids; + } + async updateVector( db: TursoDb, memoryId: string, @@ -362,7 +422,8 @@ export class TursoVectorSearch { limit: number, filters: MemoryListFilters = {} ): Promise[]> { - const conditions: string[] = []; + await this.prepareShardDb(db); + const conditions: string[] = [ACTIVE_MEMORY_SQL]; const args: Array = []; if (containerTag !== "") { conditions.push("container_tag = ?"); @@ -388,7 +449,7 @@ export class TursoVectorSearch { return db.all( ` SELECT * FROM memories - ${conditions.length > 0 ? "WHERE " + conditions.join(" AND ") : ""} + WHERE ${conditions.join(" AND ")} ORDER BY is_pinned DESC, created_at DESC LIMIT ? `, @@ -397,26 +458,38 @@ export class TursoVectorSearch { } async getAllMemories(db: TursoDb): Promise[]> { - return db.all(`SELECT * FROM memories ORDER BY created_at DESC`); + await this.prepareShardDb(db); + return db.all(`SELECT * FROM memories WHERE ${ACTIVE_MEMORY_SQL} ORDER BY created_at DESC`); } async getAllMemoriesWithExtractedVectors( db: TursoDb ): Promise & { vector_json: string | null }>> { + await this.prepareShardDb(db); return db.all(` SELECT id, content, container_tag, created_at, + is_pinned, vector_extract(vector) AS vector_json FROM memories + WHERE ${ACTIVE_MEMORY_SQL} ORDER BY created_at DESC `); } - async getMemoryById(db: TursoDb, memoryId: string): Promise | null> { - return db.get(`SELECT * FROM memories WHERE id = ?`, [memoryId]); + async getMemoryById( + db: TursoDb, + memoryId: string, + options: { includeDeleted?: boolean } = {} + ): Promise | null> { + await this.prepareShardDb(db); + if (options.includeDeleted) { + return db.get(`SELECT * FROM memories WHERE id = ?`, [memoryId]); + } + return db.get(`SELECT * FROM memories WHERE id = ? AND ${ACTIVE_MEMORY_SQL}`, [memoryId]); } async getMemoriesBySessionID(db: TursoDb, sessionID: string): Promise[]> { @@ -424,8 +497,11 @@ export class TursoVectorSearch { const rows = await db.all( ` SELECT * FROM memories - WHERE session_id = ? - OR (session_id IS NULL AND metadata LIKE ?) + WHERE ${ACTIVE_MEMORY_SQL} + AND ( + session_id = ? + OR (session_id IS NULL AND metadata LIKE ?) + ) ORDER BY created_at DESC `, [sessionID, `%"sessionID":"${sessionID}"%`] @@ -439,14 +515,15 @@ export class TursoVectorSearch { } async countVectors(db: TursoDb, containerTag: string): Promise { - const row = await db.get(`SELECT COUNT(*) as count FROM memories WHERE container_tag = ?`, [ - containerTag, - ]); + const row = await db.get( + `SELECT COUNT(*) as count FROM memories WHERE container_tag = ? AND ${ACTIVE_MEMORY_SQL}`, + [containerTag] + ); return Number(row?.count ?? 0); } async countAllVectors(db: TursoDb): Promise { - const row = await db.get(`SELECT COUNT(*) as count FROM memories`); + const row = await db.get(`SELECT COUNT(*) as count FROM memories WHERE ${ACTIVE_MEMORY_SQL}`); return Number(row?.count ?? 0); } @@ -461,6 +538,7 @@ export class TursoVectorSearch { project_name, git_repo_url FROM memories + WHERE ${ACTIVE_MEMORY_SQL} `); } @@ -474,6 +552,7 @@ export class TursoVectorSearch { COUNT(*) AS cnt FROM memories WHERE project_path IS NOT NULL AND project_path != '' + AND ${ACTIVE_MEMORY_SQL} GROUP BY project_path, container_tag ORDER BY cnt DESC, project_path ASC `); diff --git a/src/services/user-prompt/user-prompt-manager.ts b/src/services/user-prompt/user-prompt-manager.ts index 72a511e2..4ec715fc 100644 --- a/src/services/user-prompt/user-prompt-manager.ts +++ b/src/services/user-prompt/user-prompt-manager.ts @@ -144,6 +144,23 @@ export class UserPromptManager { return row ? this.rowToPrompt(row) : null; } + /** Latest saved prompt text for a session (any capture state), for relevance recall. */ + async getLatestPromptTextForSession(sessionId: string): Promise { + const db = await this.ready(); + const row = await db.get( + ` + SELECT content FROM user_prompts + WHERE session_id = ? + ORDER BY created_at DESC + LIMIT 1 + `, + [sessionId] + ); + if (!row?.content) return null; + const text = String(row.content).trim(); + return text.length > 0 ? text : null; + } + async getUncapturedPromptsForSession(sessionId: string): Promise { const db = await this.ready(); const maxRetries = CONFIG.autoCaptureMaxRetries ?? 3; diff --git a/src/services/web-server.ts b/src/services/web-server.ts index 342e4ca5..e2050591 100644 --- a/src/services/web-server.ts +++ b/src/services/web-server.ts @@ -14,6 +14,8 @@ import { handleListMemories, handleAddMemory, handleDeleteMemory, + handleListTrash, + handleRestoreMemory, handleBulkDelete, handleUpdateMemory, handleSearch, @@ -22,6 +24,8 @@ import { handleUnpinMemory, handleRunCleanup, handleRunDeduplication, + handleDreamingStatus, + handleRunDreaming, handleDetectMigration, handleRunMigration, handleDetectTagMigration, @@ -533,10 +537,26 @@ export class WebServer { return this.jsonResponse(result); } + if (path === "/api/memories/trash" && method === "GET") { + const limit = parseInt(url.searchParams.get("limit") || "200"); + const result = await handleListTrash(limit); + return this.jsonResponse(result); + } + + if (path.startsWith("/api/memories/") && path.endsWith("/restore") && method === "POST") { + const parts = path.split("/"); + const id = parts[3]; + if (!id || id === "trash") { + return this.jsonResponse({ success: false, error: "Invalid ID" }); + } + const result = await handleRestoreMemory(id); + return this.jsonResponse(result); + } + if (path.startsWith("/api/memories/") && method === "DELETE") { const parts = path.split("/"); const id = parts[3]; - if (!id || id === "bulk-delete") { + if (!id || id === "bulk-delete" || id === "trash") { return this.jsonResponse({ success: false, error: "Invalid ID" }); } const cascade = url.searchParams.get("cascade") === "true"; @@ -546,7 +566,7 @@ export class WebServer { if (path.startsWith("/api/memories/") && method === "GET") { const id = path.split("/").pop(); - if (!id || id === "bulk-delete") { + if (!id || id === "bulk-delete" || id === "trash") { return this.jsonResponse({ success: false, error: "Invalid ID" }); } const result = await handleGetMemory(id); @@ -629,6 +649,16 @@ export class WebServer { return this.jsonResponse(result); } + if (path === "/api/dreaming/status" && method === "GET") { + const result = await handleDreamingStatus(); + return this.jsonResponse(result); + } + + if (path === "/api/dreaming/run" && method === "POST") { + const result = await handleRunDreaming(); + return this.jsonResponse(result); + } + if (path === "/api/migration/detect" && method === "GET") { const result = await handleDetectMigration(); return this.jsonResponse(result); diff --git a/tests/config.test.ts b/tests/config.test.ts index 98586ebc..9e7beceb 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -108,6 +108,18 @@ describe("config", () => { expect(["first", "always"]).toContain(CONFIG.chatMessage.injectOn); }); + it("should default chatMessage.recall to relevant with default scope", () => { + expect(CONFIG.chatMessage.recall).toBe("relevant"); + expect(CONFIG.chatMessage.recallScope).toBe("default"); + }); + + it("should default dreaming to disabled", () => { + expect(CONFIG.dreaming.enabled).toBe(false); + expect(CONFIG.dreaming.intervalHours).toBe(24); + expect(CONFIG.dreaming.maxGroupsPerRun).toBe(5); + expect(CONFIG.dreaming.trashRetentionDays).toBe(30); + }); + it("should have boolean toggle settings", () => { expect(typeof CONFIG.autoCaptureEnabled).toBe("boolean"); expect(typeof CONFIG.injectProfile).toBe("boolean"); diff --git a/tests/dreaming-prompt.test.ts b/tests/dreaming-prompt.test.ts new file mode 100644 index 00000000..c06226b6 --- /dev/null +++ b/tests/dreaming-prompt.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "bun:test"; +import { + normalizeDreamingIntervalHours, + normalizeDreamingMaxGroupsPerRun, + normalizeDreamingTrashRetentionDays, +} from "../src/config.js"; +import { mergeBoth, applyProjectBoost } from "../src/services/memory-recall.js"; + +describe("dreaming config helpers", () => { + it("normalizes dreaming numeric knobs", () => { + expect(normalizeDreamingIntervalHours(12)).toBe(12); + expect(normalizeDreamingIntervalHours(0)).toBe(24); + expect(normalizeDreamingMaxGroupsPerRun(2.9)).toBe(2); + expect(normalizeDreamingTrashRetentionDays(7)).toBe(7); + expect(normalizeDreamingTrashRetentionDays(-1)).toBe(30); + }); +}); + +describe("recall merge stays stable for dreaming-adjacent selection", () => { + it("keeps relevant ahead of recent fillers", () => { + const out = mergeBoth( + [{ id: "a", memory: "a", similarity: 0.9 }], + [ + { id: "a", memory: "a", similarity: 1 }, + { id: "b", memory: "b", similarity: 1 }, + ], + 2 + ); + expect(out.map((m) => m.id)).toEqual(["a", "b"]); + }); + + it("clamps boosted similarity to 1", () => { + const [hit] = applyProjectBoost( + [{ id: "x", memory: "x", similarity: 0.99, containerTag: "p1" }], + "p1", + 0.05 + ); + expect(hit!.similarity).toBe(1); + }); +}); diff --git a/tests/fixtures/child-session-capture.mjs b/tests/fixtures/child-session-capture.mjs index a24e8a71..0d395605 100644 --- a/tests/fixtures/child-session-capture.mjs +++ b/tests/fixtures/child-session-capture.mjs @@ -9,7 +9,11 @@ const config = { excludeCurrentSession: true, filterInjectedPrompts: true, captureChildSessions: false, + recall: "recent", + recallScope: "default", }, + memory: { defaultScope: "project" }, + similarityThreshold: 0.6, compaction: { enabled: false }, autoCaptureEnabled: false, webServerEnabled: false, @@ -56,6 +60,7 @@ mock.module(moduleUrl("services/user-prompt/user-prompt-manager"), () => ({ userPromptManager: { savePrompt: async (...args) => captures.push(args), setPromptModel: async () => {}, + getLatestPromptTextForSession: async () => "fixture query", }, })); mock.module(moduleUrl("services/logger"), () => ({ diff --git a/tests/fixtures/memory-injection-eligibility.mjs b/tests/fixtures/memory-injection-eligibility.mjs index 35e18eeb..c78da74c 100644 --- a/tests/fixtures/memory-injection-eligibility.mjs +++ b/tests/fixtures/memory-injection-eligibility.mjs @@ -16,7 +16,11 @@ const config = { maxMemories: 3, excludeCurrentSession: true, filterInjectedPrompts: true, + recall: "recent", + recallScope: "default", }, + memory: { defaultScope: "project" }, + similarityThreshold: 0.6, compaction: { enabled: false }, autoCaptureEnabled: false, webServerEnabled: false, @@ -64,6 +68,7 @@ mock.module(moduleUrl("services/user-prompt/user-prompt-manager"), () => ({ userPromptManager: { savePrompt: async () => {}, setPromptModel: async () => {}, + getLatestPromptTextForSession: async () => "fixture query", reset() {}, }, })); diff --git a/tests/fixtures/profile-inject-empty-memories.mjs b/tests/fixtures/profile-inject-empty-memories.mjs index aceba51c..041db201 100644 --- a/tests/fixtures/profile-inject-empty-memories.mjs +++ b/tests/fixtures/profile-inject-empty-memories.mjs @@ -10,7 +10,11 @@ const config = { excludeCurrentSession: false, filterInjectedPrompts: true, captureChildSessions: false, + recall: "recent", + recallScope: "default", }, + memory: { defaultScope: "project" }, + similarityThreshold: 0.6, compaction: { enabled: false }, autoCaptureEnabled: false, webServerEnabled: false, @@ -60,6 +64,7 @@ mock.module(moduleUrl("services/user-prompt/user-prompt-manager"), () => ({ userPromptManager: { savePrompt: async () => {}, setPromptModel: async () => {}, + getLatestPromptTextForSession: async () => null, }, })); mock.module(moduleUrl("services/logger"), () => ({ diff --git a/tests/fixtures/v2-memory-context.mjs b/tests/fixtures/v2-memory-context.mjs index 1f360997..10a81831 100644 --- a/tests/fixtures/v2-memory-context.mjs +++ b/tests/fixtures/v2-memory-context.mjs @@ -8,7 +8,11 @@ const config = { maxMemories: 3, excludeCurrentSession: true, filterInjectedPrompts: true, + recall: "recent", + recallScope: "default", }, + memory: { defaultScope: "project" }, + similarityThreshold: 0.6, compaction: { enabled: false }, autoCaptureEnabled: false, webServerEnabled: false, @@ -64,6 +68,7 @@ mock.module(moduleUrl("services/user-prompt/user-prompt-manager"), () => ({ userPromptManager: { savePrompt: async (...args) => captures.push(args), setPromptModel: async () => {}, + getLatestPromptTextForSession: async () => "fixture query", }, })); mock.module(moduleUrl("services/logger"), () => ({ diff --git a/tests/markdown-write-through.test.ts b/tests/markdown-write-through.test.ts index a5bcbce3..a5ef3055 100644 --- a/tests/markdown-write-through.test.ts +++ b/tests/markdown-write-through.test.ts @@ -78,7 +78,11 @@ describe("markdown write-through", () => { const deleted = await memoryClient.deleteMemory(added.id); expect(deleted.success).toBe(true); - expect(existsSync(filePath)).toBe(false); + // Soft-delete keeps the markdown file so trash restore can re-surface it. + expect(existsSync(filePath)).toBe(true); + const listed = await memoryClient.listMemories(tag.tag, 20, "project"); + expect(listed.success).toBe(true); + expect(listed.memories.some((m: { id: string }) => m.id === added.id)).toBe(false); }); it("reconcile indexes foreign markdown pages", async () => { diff --git a/tests/memory-recall.test.ts b/tests/memory-recall.test.ts new file mode 100644 index 00000000..0c87de4b --- /dev/null +++ b/tests/memory-recall.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "bun:test"; +import { + applyProjectBoost, + filterRecallResults, + mergeBoth, + resolveRecallScope, + shouldRefreshRecallOnPrompt, + type RecallMemory, +} from "../src/services/memory-recall.js"; +import { normalizeChatMessageRecall, normalizeChatMessageRecallScope } from "../src/config.js"; + +function mem(partial: Partial & { id: string }): RecallMemory { + return { + memory: partial.memory ?? `content-${partial.id}`, + similarity: partial.similarity ?? 0.8, + ...partial, + }; +} + +describe("memory-recall helpers", () => { + it("refreshes every turn for relevant/both even when injectOn is first", () => { + expect(shouldRefreshRecallOnPrompt("relevant", "first")).toBe(true); + expect(shouldRefreshRecallOnPrompt("both", "first")).toBe(true); + expect(shouldRefreshRecallOnPrompt("recent", "first")).toBe(false); + expect(shouldRefreshRecallOnPrompt("recent", "always")).toBe(true); + }); + + it("resolves recallScope default from memory.defaultScope", () => { + expect(resolveRecallScope("default", "project")).toBe("project"); + expect(resolveRecallScope("default", "all-projects")).toBe("all-projects"); + expect(resolveRecallScope("project", "all-projects")).toBe("project"); + expect(resolveRecallScope("all-projects", "project")).toBe("all-projects"); + }); + + it("normalizes recall config with safe fallbacks", () => { + expect(normalizeChatMessageRecall("both")).toBe("both"); + expect(normalizeChatMessageRecall("nope")).toBe("relevant"); + expect(normalizeChatMessageRecallScope("all-projects")).toBe("all-projects"); + expect(normalizeChatMessageRecallScope(42)).toBe("default"); + }); + + it("boosts current-project hits then re-sorts", () => { + const tag = "opencode_project_abc"; + const boosted = applyProjectBoost( + [ + mem({ id: "other", similarity: 0.9, containerTag: "opencode_project_other" }), + mem({ id: "cur", similarity: 0.88, containerTag: tag }), + ], + tag + ); + expect(boosted[0]!.id).toBe("cur"); + expect(boosted[0]!.similarity).toBeCloseTo(0.93); + expect(boosted[1]!.similarity).toBe(0.9); + }); + + it("filters by session and age", () => { + const now = Date.now(); + const filtered = filterRecallResults( + [ + mem({ id: "a", sessionId: "sess-1", createdAt: now }), + mem({ id: "b", metadata: { sessionID: "sess-2" }, createdAt: now }), + mem({ id: "c", createdAt: now - 10 * 86400000 }), + ], + { excludeSessionId: "sess-1", createdAfter: now - 5 * 86400000 } + ); + expect(filtered.map((m) => m.id)).toEqual(["b"]); + }); + + it("merges both: relevant first then recent with id dedupe", () => { + const merged = mergeBoth( + [mem({ id: "r1", similarity: 0.9 }), mem({ id: "shared", similarity: 0.85 })], + [mem({ id: "shared", similarity: 1 }), mem({ id: "n1", similarity: 1 }), mem({ id: "n2" })], + 3 + ); + expect(merged.map((m) => m.id)).toEqual(["r1", "shared", "n1"]); + expect(merged[1]!.similarity).toBe(0.85); + }); +}); diff --git a/tests/memory-trash.test.ts b/tests/memory-trash.test.ts new file mode 100644 index 00000000..bc609258 --- /dev/null +++ b/tests/memory-trash.test.ts @@ -0,0 +1,129 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { tryAcquireDreamingLock } from "../src/services/dreaming-lock.js"; +import { cleanupTursoTestDirectory } from "./turso-test-utils.js"; + +describe("dreaming lock", () => { + let baseDir: string | undefined; + let previousStorage: string | undefined; + + afterEach(async () => { + if (previousStorage !== undefined) { + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = previousStorage; + } + if (baseDir) await cleanupTursoTestDirectory(baseDir); + baseDir = undefined; + }); + + it("allows only one holder at a time and releases cleanly", async () => { + baseDir = mkdtempSync(join(tmpdir(), "dream-lock-")); + const { CONFIG } = await import("../src/config.js"); + previousStorage = CONFIG.storagePath; + CONFIG.storagePath = baseDir; + + const first = tryAcquireDreamingLock(); + expect(first).not.toBeNull(); + const second = tryAcquireDreamingLock(); + expect(second).toBeNull(); + first!(); + const third = tryAcquireDreamingLock(); + expect(third).not.toBeNull(); + third!(); + }); +}); + +describe("memory soft-delete trash", () => { + let baseDir: string; + let restoreEmbedding: (() => void) | undefined; + let previousBackend: "database" | "markdown" | undefined; + let previousStorage: string | undefined; + let previousDims: number | undefined; + + afterEach(async () => { + restoreEmbedding?.(); + restoreEmbedding = undefined; + const { CONFIG } = await import("../src/config.js"); + if (previousBackend !== undefined) CONFIG.memoryBackend = previousBackend; + if (previousStorage !== undefined) CONFIG.storagePath = previousStorage; + if (previousDims !== undefined) CONFIG.embeddingDimensions = previousDims; + previousBackend = undefined; + previousStorage = undefined; + previousDims = undefined; + const { memoryClient } = await import("../src/services/client.js"); + memoryClient.reset(); + const { resetTursoReady } = await import("../src/services/turso/ready.js"); + resetTursoReady(); + if (baseDir) await cleanupTursoTestDirectory(baseDir); + }); + + async function stubEmbedding() { + const { embeddingService } = await import("../src/services/embedding.js"); + const service = embeddingService as any; + const original = { + warmup: service.warmup, + clearCache: service.clearCache, + embedWithTimeout: service.embedWithTimeout, + isWarmedUp: service.isWarmedUp, + }; + service.warmup = async () => { + service.isWarmedUp = true; + }; + service.clearCache = () => {}; + service.embedWithTimeout = async () => new Float32Array([1, 0]); + restoreEmbedding = () => Object.assign(service, original); + } + + it("hides soft-deleted rows from list and restores them from trash", async () => { + baseDir = mkdtempSync(join(tmpdir(), "mem-trash-")); + const storage = join(baseDir, "storage"); + + const { CONFIG } = await import("../src/config.js"); + previousBackend = CONFIG.memoryBackend; + previousStorage = CONFIG.storagePath; + previousDims = CONFIG.embeddingDimensions; + CONFIG.storagePath = storage; + CONFIG.embeddingDimensions = 2; + CONFIG.memoryBackend = "database"; + + await stubEmbedding(); + + const { memoryClient } = await import("../src/services/client.js"); + memoryClient.reset(); + + const { getProjectTagInfo } = await import("../src/services/tags.js"); + const tag = getProjectTagInfo(baseDir); + + const added = await memoryClient.addMemory("trashable memory", tag.tag, { + source: "manual", + type: "fact", + }); + expect(added.success).toBe(true); + if (!added.success) { + throw new Error(added.error || "addMemory failed"); + } + + const before = await memoryClient.listMemories(tag.tag, 10, "project"); + expect(before.memories.some((m: { id: string }) => m.id === added.id)).toBe(true); + + const deleted = await memoryClient.deleteMemory(added.id); + expect(deleted.success).toBe(true); + + const after = await memoryClient.listMemories(tag.tag, 10, "project"); + expect(after.memories.some((m: { id: string }) => m.id === added.id)).toBe(false); + + const { handleListTrash, handleRestoreMemory } = + await import("../src/services/api-handlers.js"); + const trash = await handleListTrash(); + expect(trash.success).toBe(true); + expect(trash.data?.some((row) => row.id === added.id)).toBe(true); + + const restored = await handleRestoreMemory(added.id); + expect(restored.success).toBe(true); + + const again = await memoryClient.listMemories(tag.tag, 10, "project"); + expect(again.memories.some((m: { id: string }) => m.id === added.id)).toBe(true); + }); +}); diff --git a/tests/profile-inject-empty-memories.test.ts b/tests/profile-inject-empty-memories.test.ts index c09a1365..aab79f6e 100644 --- a/tests/profile-inject-empty-memories.test.ts +++ b/tests/profile-inject-empty-memories.test.ts @@ -24,7 +24,8 @@ beforeAll(() => { describe("profile injection with empty project memories", () => { it("still formats context when chatMessage.maxMemories is 0", () => { - expect(result.listLimit).toBe(0); + // maxMemories 0 skips listing/search entirely; format still runs for profile-only inject. + expect(result.listLimit === 0 || result.listLimit === null).toBe(true); expect(result.formatCalls).toHaveLength(2); expect(result.formatCalls.every((call: any) => call.resultCount === 0)).toBe(true); expect( diff --git a/tests/schema-migrations.test.ts b/tests/schema-migrations.test.ts index 6023a868..8eae8a5a 100644 --- a/tests/schema-migrations.test.ts +++ b/tests/schema-migrations.test.ts @@ -42,8 +42,8 @@ describe("schema migrations", () => { await db.close(); }); - it("migrates memory shards to session_id index (v2)", async () => { - baseDir = mkdtempSync(join(tmpdir(), "schema-mem-v2-")); + it("migrates memory shards through session_id (v2) and deleted_at (v3)", async () => { + baseDir = mkdtempSync(join(tmpdir(), "schema-mem-v3-")); const dbPath = join(baseDir, "shard.db"); const native = await connect(dbPath); const db = new TursoDb(native); @@ -74,13 +74,14 @@ describe("schema migrations", () => { dbPath, label: "memory-shard", }); - expect(version).toBe(2); + expect(version).toBe(3); - const row = await db.get<{ session_id: string }>( - `SELECT session_id FROM memories WHERE id = ?`, + const row = await db.get<{ session_id: string; deleted_at: number | null }>( + `SELECT session_id, deleted_at FROM memories WHERE id = ?`, ["mem_legacy"] ); expect(row?.session_id).toBe("sess_from_meta"); + expect(row?.deleted_at ?? null).toBeNull(); await db.close(); }); }); diff --git a/web/src/App.svelte b/web/src/App.svelte index 3ff888ee..2ea8a071 100644 --- a/web/src/App.svelte +++ b/web/src/App.svelte @@ -18,6 +18,7 @@ import MarkdownVaultView from "$lib/components/explorer/MarkdownVaultView.svelte"; import StatsView from "$lib/components/explorer/StatsView.svelte"; import TagMigrationDialog from "$lib/components/explorer/TagMigrationDialog.svelte"; + import TrashDialog from "$lib/components/explorer/TrashDialog.svelte"; import TagFilter from "$lib/components/explorer/TagFilter.svelte"; import Alert from "$lib/components/ui/alert.svelte"; import AlertDescription from "$lib/components/ui/alert-description.svelte"; @@ -343,6 +344,12 @@