From d72bf6bd600e4029cc44369b19390820cb4dd57c Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:55:36 +0100 Subject: [PATCH 1/6] feat(capture): optional update-on-capture for similar memories Search for similar memories before auto-capture persist and let the model choose add, update, or supersede (soft-delete into trash). --- README.md | 34 +++- opencode-mem.example.jsonc | 13 +- src/config.ts | 72 +++++++- src/services/api-handlers.ts | 103 +---------- src/services/auto-capture.ts | 229 ++++++++++++++++++++++++- src/services/client.ts | 161 ++++++++++++++++- src/services/update-on-capture.ts | 186 ++++++++++++++++++++ tests/add-memory-phase-logging.test.ts | 1 + tests/config.test.ts | 8 + tests/update-on-capture.test.ts | 152 ++++++++++++++++ 10 files changed, 843 insertions(+), 116 deletions(-) create mode 100644 src/services/update-on-capture.ts create mode 100644 tests/update-on-capture.test.ts diff --git a/README.md b/README.md index 6515cab0..5d2500af 100644 --- a/README.md +++ b/README.md @@ -279,14 +279,36 @@ memories do not consume slots that could hold eligible memories. Setting `chatMessage.maxMemories` to `0` disables project-memory injection while still allowing the user profile to be injected when enabled. +### Update-on-capture (optional) + +Opt-in path that keeps memories current as facts change. When `updateOnCaptureEnabled` is +`true`, auto-capture embeds the new summary, searches for similar memories above +`updateOnCaptureSimilarityThreshold` (default `0.75`), and asks the capture model to +choose `add`, `update` (rewrite in place), or `supersede` (save new + soft-delete old into +trash). Pinned memories are skipped. Default is off so existing installs keep add-only +behavior. + +```jsonc +"updateOnCaptureEnabled": true, +"updateOnCaptureSimilarityThreshold": 0.75, +"updateOnCaptureMaxCandidates": 5, +``` + +Tip: with `Xenova/nomic-embed-text-v1`, enabling `embeddingUseTaskPrefixes: true` (and +optionally lowering recall `similarityThreshold`) often improves match quality for longer +memories. Prefixes stay opt-in by default. + ### Dreaming (optional memory consolidation) -Opt-in background pass that finds near-duplicate memories (same similarity threshold as -deduplication), asks the configured `opencodeProvider` / `opencodeModel` to merge them, -and soft-deletes superseded rows into recoverable trash. Pinned memories are never -touched. Configure under `dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). -The web UI exposes **Trash** (restore) and **Dream** (manual run). Soft-deleted rows are -hard-purged after `dreaming.trashRetentionDays` during normal cleanup. +Opt-in background pass that finds **related** memories (via `dreaming.relatedThreshold`, +default `0.75`, independent of `deduplicationSimilarityThreshold`), asks the configured +`opencodeProvider` / `opencodeModel` to `merge`, `update`, `supersede`, `keep`, or `flag` +contradictions, and soft-deletes superseded rows into recoverable trash. Pinned memories +are never touched. Set `dreaming.crossProject: true` (or `memory.defaultScope: +"all-projects"`) to group across project tags; user-scope stays separate. Configure under +`dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). The web UI exposes +**Trash** (restore) and **Dream** (manual run). Soft-deleted rows are hard-purged after +`dreaming.trashRetentionDays` during normal cleanup. ### Choosing / configuring embeddings diff --git a/opencode-mem.example.jsonc b/opencode-mem.example.jsonc index 801c7027..fd61a6c6 100644 --- a/opencode-mem.example.jsonc +++ b/opencode-mem.example.jsonc @@ -155,13 +155,16 @@ // "injectionMarkers": [""], }, - // Opt-in background consolidation ("dreaming"): merge near-duplicates via LLM - // and soft-delete superseded memories into recoverable trash. + // Opt-in background consolidation ("dreaming"): merge/update/supersede related + // memories via LLM and soft-delete superseded rows into recoverable trash. "dreaming": { "enabled": false, "intervalHours": 24, "maxGroupsPerRun": 5, "trashRetentionDays": 30, + "relatedThreshold": 0.75, + // Also effective when memory.defaultScope is "all-projects". + "crossProject": false, }, // ============================================ @@ -205,6 +208,12 @@ "autoCaptureEnabled": true, + // Opt-in: before saving a new auto-capture memory, search for similar entries + // and let the model choose add / update / supersede (old rows go to trash). + // "updateOnCaptureEnabled": true, + // "updateOnCaptureSimilarityThreshold": 0.75, + // "updateOnCaptureMaxCandidates": 5, + // When both opencodeProvider and manual memoryModel/memoryApiUrl are set, which // channel to try first for auto-capture summaries. Default "opencode-first". // Prefer "external-first" if the host session API is slow/busy and you want the diff --git a/src/config.ts b/src/config.ts index 2917e83b..1dd6e666 100644 --- a/src/config.ts +++ b/src/config.ts @@ -81,6 +81,16 @@ interface OpenCodeMemConfig { autoCaptureMaxRetries?: number; autoCaptureMaxContextBytes?: number; autoCaptureLanguage?: string; + /** + * When true, auto-capture searches for similar memories before insert and + * may update or supersede an existing entry instead of always adding. + * Default false (opt-in). + */ + updateOnCaptureEnabled?: boolean; + /** Cosine/hybrid floor for update-on-capture candidates. Default 0.75. */ + updateOnCaptureSimilarityThreshold?: number; + /** Max similar memories shown to the decision LLM. Default 5. */ + updateOnCaptureMaxCandidates?: number; /** * Which auto-capture summary channel to try first when both opencode and * manual (memoryModel/memoryApiUrl) providers are configured. @@ -239,6 +249,16 @@ interface OpenCodeMemConfig { intervalHours?: number; maxGroupsPerRun?: number; trashRetentionDays?: number; + /** + * Similarity floor for related (not only near-duplicate) dreaming groups. + * Default 0.75. Independent of deduplicationSimilarityThreshold. + */ + relatedThreshold?: number; + /** + * Allow grouping across project container tags. Also effective when + * memory.defaultScope is "all-projects". User-scope stays separate. + */ + crossProject?: boolean; }; } @@ -326,6 +346,9 @@ const DEFAULTS: Required< autoCaptureMaxRetries: 3, autoCaptureMaxContextBytes: 131072, autoCaptureProviderOrder: "opencode-first", + updateOnCaptureEnabled: false, + updateOnCaptureSimilarityThreshold: 0.75, + updateOnCaptureMaxCandidates: 5, aiSessionRetentionDays: 7, webServerEnabled: true, webServerPort: 4747, @@ -389,6 +412,8 @@ const DEFAULTS: Required< intervalHours: 24, maxGroupsPerRun: 5, trashRetentionDays: 30, + relatedThreshold: 0.75, + crossProject: false, }, }; @@ -600,6 +625,12 @@ const CONFIG_TEMPLATE = `{ // Note: Ollama may not support tool calling. Use OpenAI, Anthropic, or Groq for best results. "autoCaptureEnabled": true, + + // Opt-in: before saving a new auto-capture memory, search for similar entries + // and let the model choose add / update / supersede (old rows go to trash). + // "updateOnCaptureEnabled": true, + // "updateOnCaptureSimilarityThreshold": 0.75, + // "updateOnCaptureMaxCandidates": 5, // Provider type: "api-route" | "atlas-cloud" | "cheaper-inference" | "openai-chat" | "openai-responses" | "anthropic" | "minimax" | "orcarouter" | "runcrate" // Note: "openai-chat" is a generic OpenAI API-compatible mode. @@ -890,13 +921,18 @@ const CONFIG_TEMPLATE = `{ // "injectionMarkers": [""] }, - // Opt-in background consolidation: merge near-duplicate memories via the - // configured LLM and soft-delete superseded rows into trash (recoverable). + // Opt-in background consolidation: merge/update/supersede related memories + // via the configured LLM and soft-delete superseded rows into trash. "dreaming": { "enabled": false, "intervalHours": 24, "maxGroupsPerRun": 5, - "trashRetentionDays": 30 + "trashRetentionDays": 30, + // Related (not only near-duplicate) grouping floor; independent of + // deduplicationSimilarityThreshold (default 0.9). + "relatedThreshold": 0.75, + // Also effective when memory.defaultScope is "all-projects". + "crossProject": false } } `; @@ -1158,6 +1194,26 @@ export function normalizeDreamingTrashRetentionDays(value: unknown): number { return value; } +export function normalizeSimilarityUnitInterval(value: unknown, fallback: number): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + return fallback; + } + if (value < 0) return 0; + if (value > 1) return 1; + return value; +} + +export function normalizeUpdateOnCaptureMaxCandidates(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return 5; + } + return Math.max(1, Math.min(20, Math.trunc(value))); +} + +export function normalizeDreamingRelatedThreshold(value: unknown): number { + return normalizeSimilarityUnitInterval(value, 0.75); +} + /** * User-supplied markers extend the built-in set rather than replacing it, so * adding one marker cannot silently disable protection against all the others. @@ -1288,6 +1344,14 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { autoCaptureProviderOrder: normalizeAutoCaptureProviderOrder( fileConfig.autoCaptureProviderOrder ), + updateOnCaptureEnabled: fileConfig.updateOnCaptureEnabled ?? DEFAULTS.updateOnCaptureEnabled, + updateOnCaptureSimilarityThreshold: normalizeSimilarityUnitInterval( + fileConfig.updateOnCaptureSimilarityThreshold, + DEFAULTS.updateOnCaptureSimilarityThreshold + ), + updateOnCaptureMaxCandidates: normalizeUpdateOnCaptureMaxCandidates( + fileConfig.updateOnCaptureMaxCandidates + ), memoryProvider, memoryModel: fileConfig.memoryModel, memoryApiUrl: fileConfig.memoryApiUrl, @@ -1416,6 +1480,8 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { trashRetentionDays: normalizeDreamingTrashRetentionDays( fileConfig.dreaming?.trashRetentionDays ), + relatedThreshold: normalizeDreamingRelatedThreshold(fileConfig.dreaming?.relatedThreshold), + crossProject: fileConfig.dreaming?.crossProject ?? DEFAULTS.dreaming.crossProject, }, }; } diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 05f70a43..0ddc260a 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -3,7 +3,7 @@ import { tursoShardManager } from "./turso/shard-manager.js"; import { tursoVectorSearch } from "./turso/vector-search.js"; import { tursoConnectionManager } from "./turso/connection-manager.js"; import { ensureTursoReady } from "./turso/ready.js"; -import { formatTagsForEmbedding, parseSessionIdFromMetadata } from "./turso/vector-utils.js"; +import { formatTagsForEmbedding } from "./turso/vector-utils.js"; import { extractScopeFromContainerTag, tryExtractScopeFromContainerTag } from "./memory-scope.js"; import { log } from "./logger.js"; import { CONFIG } from "../config.js"; @@ -592,104 +592,11 @@ export async function handleUpdateMemory( data: { content?: string; type?: MemoryType; tags?: string[] } ): Promise> { try { - if (!id) return { success: false, error: "id is required" }; - await embeddingService.warmup(); - - // Find the existing memory first (read-only — no data modified yet) - const shards = await getAllMemoryShards(); - let foundShard = null, - existingMemory = null; - for (const shard of shards) { - const db = await tursoConnectionManager.getConnection(shard.dbPath); - const memory = await tursoVectorSearch.getMemoryById(db, id); - if (memory) { - foundShard = shard; - existingMemory = memory; - break; - } + const { memoryClient } = await import("./client.js"); + const result = await memoryClient.updateMemory(id, data); + if (!result.success) { + return { success: false, error: result.error || "Update failed" }; } - if (!foundShard || !existingMemory) return { success: false, error: "Memory not found" }; - - // STEP 1: Generate new embeddings FIRST (safe — no data deleted yet) - const newContent = data.content || String(existingMemory.content); - const existingTags = existingMemory.tags ? String(existingMemory.tags) : ""; - const tags = - data.tags || (existingTags ? existingTags.split(",").map((t: string) => t.trim()) : []); - const vector = await embeddingService.embedWithTimeout(newContent, { task: "document" }); - let tagsVector: Float32Array | undefined = undefined; - if (tags.length > 0) { - tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags), { - task: "document", - }); - } - - const db = await tursoConnectionManager.getConnection(foundShard.dbPath); - const updatedAt = Date.now(); - const memoryType = data.type || (existingMemory.type ? String(existingMemory.type) : undefined); - const containerTag = String(existingMemory.container_tag); - const existingMeta = safeJSONParse(existingMemory.metadata) || {}; - - if (isMarkdownBackend()) { - writeMemoryMarkdownFile({ - id, - content: newContent, - type: memoryType, - tags, - containerTag, - createdAt: Number(existingMemory.created_at), - updatedAt, - source: typeof existingMeta.source === "string" ? existingMeta.source : "api", - title: existingMemory.display_name ? String(existingMemory.display_name) : undefined, - }); - existingMeta.markdownRelPath = pluginMemoryRelPath(id); - } - - const metadataJson = JSON.stringify(existingMeta); - - const isPinned = Number(existingMemory.is_pinned ?? 0) === 1 ? 1 : 0; - const deletedAt = - existingMemory.deleted_at == null || existingMemory.deleted_at === undefined - ? null - : Number(existingMemory.deleted_at); - - await db.transaction("write", async (tx) => { - await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [id] }); - await tx.execute({ - sql: ` - INSERT INTO memories ( - id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, - metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url, - is_pinned, deleted_at - ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `, - args: [ - id, - newContent, - JSON.stringify(Array.from(vector)), - ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), - containerTag, - tags.length > 0 ? tags.join(",") : null, - memoryType || null, - Number(existingMemory.created_at), - updatedAt, - metadataJson, - existingMemory.session_id - ? String(existingMemory.session_id) - : parseSessionIdFromMetadata( - existingMemory.metadata ? String(existingMemory.metadata) : null - ), - existingMemory.display_name ? String(existingMemory.display_name) : null, - existingMemory.user_name ? String(existingMemory.user_name) : null, - existingMemory.user_email ? String(existingMemory.user_email) : null, - existingMemory.project_path ? String(existingMemory.project_path) : null, - existingMemory.project_name ? String(existingMemory.project_name) : null, - existingMemory.git_repo_url ? String(existingMemory.git_repo_url) : null, - isPinned, - deletedAt, - ], - }); - }); - return { success: true }; } catch (error) { log("handleUpdateMemory: error", { error: String(error) }); diff --git a/src/services/auto-capture.ts b/src/services/auto-capture.ts index 2987e1cd..559ea4b6 100644 --- a/src/services/auto-capture.ts +++ b/src/services/auto-capture.ts @@ -7,6 +7,13 @@ import { CONFIG } from "../config.js"; import { userPromptManager, type UserPrompt } from "./user-prompt/user-prompt-manager.js"; import { loadOpencodeProvider } from "./ai/opencode-provider-loader.js"; import { truncateToMaxBytes, utf8ByteLength } from "../utils/context-limit.js"; +import { + applyUpdateOnCaptureDecision, + buildUpdateOnCaptureDecisionPrompt, + filterUpdateOnCaptureCandidates, + type UpdateOnCaptureCandidate, + type UpdateOnCaptureDecision, +} from "./update-on-capture.js"; interface ToolCallInfo { name: string; @@ -168,8 +175,8 @@ async function capturePrompt( : summaryResult.summary; phase = "persist"; - const result = await memoryClient.addMemory(summaryWithTags, tags.project.tag, { - source: "auto-capture" as any, + const persistMeta = { + source: "auto-capture" as const, type: summaryResult.type as any, tags: summaryResult.tags, sessionID, @@ -181,16 +188,28 @@ async function capturePrompt( projectPath: tags.project.projectPath, projectName: tags.project.projectName, gitRepoUrl: tags.project.gitRepoUrl, - }); + }; + + const result = await persistAutoCaptureMemory( + ctx, + sessionID, + prompt, + summaryWithTags, + summaryResult.type, + summaryResult.tags || [], + tags.project.tag, + persistMeta + ); if (result.success) { - await userPromptManager.linkMemoryToPrompt(prompt.id, result.id); + await userPromptManager.linkMemoryToPrompt(prompt.id, result.id!); await userPromptManager.markAsCaptured(prompt.id); claimedPromptId = null; log("Auto-capture memory persisted", { promptId: prompt.id, sessionID, memoryId: result.id, + action: result.action, }); if (CONFIG.showAutoCaptureToasts) { @@ -712,6 +731,208 @@ async function tryExternalSummary( }; } +async function persistAutoCaptureMemory( + ctx: PluginInput, + sessionID: string, + prompt: UserPrompt, + summaryWithTags: string, + type: string, + tags: string[], + containerTag: string, + metadata: Record +): Promise<{ success: boolean; id?: string; error?: string; action: string }> { + if (!CONFIG.updateOnCaptureEnabled) { + const result = await memoryClient.addMemory(summaryWithTags, containerTag, metadata as any); + return { + success: result.success, + id: result.success ? result.id : undefined, + error: result.success ? undefined : result.error, + action: "add", + }; + } + + const scope = CONFIG.memory.defaultScope === "all-projects" ? "all-projects" : "project"; + const search = await memoryClient.searchMemories(summaryWithTags, containerTag, scope, { + similarityThreshold: CONFIG.updateOnCaptureSimilarityThreshold, + limit: CONFIG.updateOnCaptureMaxCandidates, + }); + + const rawCandidates: UpdateOnCaptureCandidate[] = (search.success ? search.results : []).map( + (r: any) => ({ + id: String(r.id), + content: String(r.memory ?? r.content ?? ""), + similarity: Number(r.similarity ?? 0), + isPinned: Number(r.isPinned ?? 0) === 1 || r.isPinned === true, + }) + ); + const candidates = filterUpdateOnCaptureCandidates(rawCandidates); + + if (candidates.length === 0) { + const result = await memoryClient.addMemory(summaryWithTags, containerTag, metadata as any); + return { + success: result.success, + id: result.success ? result.id : undefined, + error: result.success ? undefined : result.error, + action: "add", + }; + } + + let decision: UpdateOnCaptureDecision; + try { + decision = await decideUpdateOnCapture(ctx, sessionID, prompt, summaryWithTags, candidates); + } catch (error) { + log("update-on-capture: decision failed, falling back to add", serializeError(error)); + decision = { action: "add" }; + } + + return applyUpdateOnCaptureDecision( + memoryClient as any, + decision, + { + summary: summaryWithTags, + type, + tags, + containerTag, + metadata: metadata as any, + }, + new Set(candidates.map((c) => c.id)) + ); +} + +async function decideUpdateOnCapture( + ctx: PluginInput, + sessionID: string, + prompt: UserPrompt, + summary: string, + candidates: UpdateOnCaptureCandidate[] +): Promise { + const order = CONFIG.autoCaptureProviderOrder ?? "opencode-first"; + const channels: CaptureChannel[] = + order === "external-first" ? ["external", "opencode"] : ["opencode", "external"]; + + let lastError: unknown; + for (let i = 0; i < channels.length; i++) { + const channel = channels[i]!; + if (!isChannelConfigured(channel)) continue; + try { + return channel === "opencode" + ? await tryOpencodeUpdateDecision(summary, candidates, prompt) + : await tryExternalUpdateDecision(summary, candidates, sessionID, prompt); + } catch (error) { + lastError = error; + const nextConfigured = channels.slice(i + 1).find((c) => isChannelConfigured(c)); + if (!nextConfigured) break; + showFallbackToast(ctx, channel, error); + } + } + if (lastError) throw lastError; + return { action: "add" }; +} + +async function tryOpencodeUpdateDecision( + summary: string, + candidates: UpdateOnCaptureCandidate[], + prompt?: { id?: string; providerId: string | null; modelId: string | null } +): Promise { + const { ensureProviderConnected, getV2Client, generateStructuredOutput } = + await loadOpencodeProvider(); + + let providerID = CONFIG.opencodeProvider!; + let modelID = CONFIG.opencodeModel!; + if (modelID === "inherit") { + if (!prompt?.providerId || !prompt?.modelId) { + throw new Error( + "opencode-mem: opencodeModel is 'inherit' but no session model was recorded for this prompt" + ); + } + providerID = prompt.providerId; + modelID = prompt.modelId; + } + + if (!(await ensureProviderConnected(providerID))) { + throw new Error(`opencode provider '${providerID}' is not connected`); + } + const v2Client = getV2Client(); + if (!v2Client) { + throw new Error("opencode-mem: v2 client not initialized"); + } + + const { z } = await import("zod"); + const schema = z.object({ + action: z.enum(["add", "update", "supersede"]), + targetId: z.string().optional(), + content: z.string().optional(), + reason: z.string().optional(), + }); + + return generateStructuredOutput({ + client: v2Client, + providerID, + modelID, + ...(CONFIG.opencodeVariant ? { variant: CONFIG.opencodeVariant } : {}), + systemPrompt: + "You decide whether a new memory should add, update, or supersede similar existing memories. Return only structured JSON.", + userPrompt: buildUpdateOnCaptureDecisionPrompt(summary, candidates), + schema, + }); +} + +async function tryExternalUpdateDecision( + summary: string, + candidates: UpdateOnCaptureCandidate[], + sessionID: string, + prompt?: { id?: string; providerId: string | null; modelId: string | null } +): Promise { + const { AIProviderFactory } = await import("./ai/ai-provider-factory.js"); + const { buildMemoryProviderConfig } = await import("./ai/provider-config.js"); + + const providerConfig = buildMemoryProviderConfig(CONFIG); + const provider = AIProviderFactory.createProvider(CONFIG.memoryProvider, providerConfig); + const toolSchema = { + type: "function" as const, + function: { + name: "decide_memory_write", + description: "Decide how to store a new memory relative to similar existing ones", + parameters: { + type: "object", + properties: { + action: { + type: "string", + enum: ["add", "update", "supersede"], + }, + targetId: { type: "string" }, + content: { type: "string" }, + reason: { type: "string" }, + }, + required: ["action"], + }, + }, + }; + + const captureSessionID = `update-on-capture-${prompt?.id ?? sessionID}-${randomUUID()}`; + const result = await provider.executeToolCall( + "You decide whether a new memory should add, update, or supersede similar existing memories.", + buildUpdateOnCaptureDecisionPrompt(summary, candidates), + toolSchema, + captureSessionID + ); + + if (!result.success || !result.data) { + throw new Error(result.error || "Failed to decide update-on-capture action"); + } + + const action = result.data.action; + if (action !== "add" && action !== "update" && action !== "supersede") { + return { action: "add" }; + } + return { + action, + targetId: typeof result.data.targetId === "string" ? result.data.targetId : undefined, + content: typeof result.data.content === "string" ? result.data.content : undefined, + reason: typeof result.data.reason === "string" ? result.data.reason : undefined, + }; +} + async function generateSummary( ctx: PluginInput, context: string, diff --git a/src/services/client.ts b/src/services/client.ts index 12a00d1a..5409396b 100644 --- a/src/services/client.ts +++ b/src/services/client.ts @@ -3,7 +3,7 @@ import { tursoShardManager } from "./turso/shard-manager.js"; import { tursoVectorSearch } from "./turso/vector-search.js"; import { tursoConnectionManager } from "./turso/connection-manager.js"; import { ensureTursoReady } from "./turso/ready.js"; -import { formatTagsForEmbedding } from "./turso/vector-utils.js"; +import { formatTagsForEmbedding, parseSessionIdFromMetadata } from "./turso/vector-utils.js"; import { extractScopeFromContainerTag, resolveMemoryScope, @@ -279,7 +279,128 @@ export class LocalMemoryClient { } } - async deleteMemory(memoryId: string) { + async updateMemory( + id: string, + data: { + content?: string; + type?: MemoryType; + tags?: string[]; + metadataPatch?: Record; + } + ) { + try { + if (!id) return { success: false as const, error: "id is required" }; + await this.initialize(); + await embeddingService.warmup(); + + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); + const shards = [...userShards, ...projectShards]; + + let foundShard = null; + let existingMemory = null; + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id); + if (memory) { + foundShard = shard; + existingMemory = memory; + break; + } + } + if (!foundShard || !existingMemory) { + return { success: false as const, error: "Memory not found" }; + } + + const newContent = data.content || String(existingMemory.content); + const existingTags = existingMemory.tags ? String(existingMemory.tags) : ""; + const tags = + data.tags || (existingTags ? existingTags.split(",").map((t: string) => t.trim()) : []); + const vector = await embeddingService.embedWithTimeout(newContent, { task: "document" }); + let tagsVector: Float32Array | undefined = undefined; + if (tags.length > 0) { + tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags), { + task: "document", + }); + } + + const db = await tursoConnectionManager.getConnection(foundShard.dbPath); + const updatedAt = Date.now(); + const memoryType = + data.type || (existingMemory.type ? String(existingMemory.type) : undefined); + const containerTag = String(existingMemory.container_tag); + const existingMeta = safeJSONParse(existingMemory.metadata) || {}; + if (data.metadataPatch) Object.assign(existingMeta, data.metadataPatch); + + if (isMarkdownBackend()) { + writeMemoryMarkdownFile({ + id, + content: newContent, + type: memoryType, + tags, + containerTag, + createdAt: Number(existingMemory.created_at), + updatedAt, + source: typeof existingMeta.source === "string" ? existingMeta.source : "api", + title: existingMemory.display_name ? String(existingMemory.display_name) : undefined, + }); + existingMeta.markdownRelPath = pluginMemoryRelPath(id); + } + + const metadataJson = JSON.stringify(existingMeta); + const isPinned = Number(existingMemory.is_pinned ?? 0) === 1 ? 1 : 0; + const deletedAt = + existingMemory.deleted_at == null || existingMemory.deleted_at === undefined + ? null + : Number(existingMemory.deleted_at); + + await db.transaction("write", async (tx) => { + await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [id] }); + await tx.execute({ + sql: ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url, + is_pinned, deleted_at + ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + args: [ + id, + newContent, + JSON.stringify(Array.from(vector)), + ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), + containerTag, + tags.length > 0 ? tags.join(",") : null, + memoryType || null, + Number(existingMemory.created_at), + updatedAt, + metadataJson, + existingMemory.session_id + ? String(existingMemory.session_id) + : parseSessionIdFromMetadata( + existingMemory.metadata ? String(existingMemory.metadata) : null + ), + existingMemory.display_name ? String(existingMemory.display_name) : null, + existingMemory.user_name ? String(existingMemory.user_name) : null, + existingMemory.user_email ? String(existingMemory.user_email) : null, + existingMemory.project_path ? String(existingMemory.project_path) : null, + existingMemory.project_name ? String(existingMemory.project_name) : null, + existingMemory.git_repo_url ? String(existingMemory.git_repo_url) : null, + isPinned, + deletedAt, + ], + }); + }); + + return { success: true as const }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + log("updateMemory: error", { id, error: errorMessage }); + return { success: false as const, error: errorMessage }; + } + } + + async deleteMemory(memoryId: string, metaPatch?: Record) { try { await this.initialize(); @@ -293,7 +414,7 @@ export class LocalMemoryClient { if (memory) { // Soft-delete into trash (recoverable). Hard-purge happens via cleanup. - const ok = await tursoVectorSearch.softDeleteVector(db, memoryId); + const ok = await tursoVectorSearch.softDeleteVector(db, memoryId, metaPatch); return ok ? { success: true } : { success: false, error: "Memory not found" }; } } @@ -306,6 +427,40 @@ export class LocalMemoryClient { } } + async getMemoryById(memoryId: string) { + try { + await this.initialize(); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); + for (const shard of [...userShards, ...projectShards]) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, memoryId); + if (memory) { + return { + success: true as const, + memory: { + id: String(memory.id), + content: String(memory.content), + isPinned: Number(memory.is_pinned ?? 0) === 1, + containerTag: String(memory.container_tag), + type: memory.type ? String(memory.type) : undefined, + tags: memory.tags + ? String(memory.tags) + .split(",") + .map((t) => t.trim()) + .filter(Boolean) + : [], + }, + }; + } + } + return { success: false as const, error: "Memory not found" }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { success: false as const, error: errorMessage }; + } + } + async listMemories( containerTag: string, limit = 20, diff --git a/src/services/update-on-capture.ts b/src/services/update-on-capture.ts new file mode 100644 index 00000000..ee8d0489 --- /dev/null +++ b/src/services/update-on-capture.ts @@ -0,0 +1,186 @@ +import { CONFIG } from "../config.js"; +import { log } from "./logger.js"; +import type { MemoryType } from "../types/index.js"; + +export type UpdateOnCaptureAction = "add" | "update" | "supersede"; + +export interface UpdateOnCaptureCandidate { + id: string; + content: string; + similarity: number; + isPinned?: boolean; +} + +export interface UpdateOnCaptureDecision { + action: UpdateOnCaptureAction; + targetId?: string; + content?: string; + reason?: string; +} + +export type UpdateOnCaptureMemoryClient = { + addMemory: ( + content: string, + containerTag: string, + metadata?: Record + ) => Promise<{ success: boolean; id?: string; error?: string }>; + updateMemory: ( + id: string, + data: { + content?: string; + type?: MemoryType; + tags?: string[]; + metadataPatch?: Record; + } + ) => Promise<{ success: boolean; error?: string }>; + deleteMemory: ( + memoryId: string, + metaPatch?: Record + ) => Promise<{ success: boolean; error?: string }>; + getMemoryById: (memoryId: string) => Promise<{ + success: boolean; + memory?: { id: string; content: string; isPinned: boolean }; + error?: string; + }>; +}; + +export interface PersistCaptureInput { + summary: string; + type: string; + tags: string[]; + containerTag: string; + metadata: { + source: "auto-capture"; + type: MemoryType | string; + tags: string[]; + sessionID: string; + promptId: string; + captureTimestamp: number; + displayName?: string; + userName?: string; + userEmail?: string; + projectPath?: string; + projectName?: string; + gitRepoUrl?: string; + }; +} + +/** Drop pinned rows and truncate to the configured candidate cap. */ +export function filterUpdateOnCaptureCandidates( + candidates: UpdateOnCaptureCandidate[], + maxCandidates = CONFIG.updateOnCaptureMaxCandidates +): UpdateOnCaptureCandidate[] { + return candidates + .filter((c) => !c.isPinned) + .sort((a, b) => b.similarity - a.similarity) + .slice(0, Math.max(1, maxCandidates)); +} + +export function buildUpdateOnCaptureDecisionPrompt( + summary: string, + candidates: UpdateOnCaptureCandidate[] +): string { + const items = candidates.map((c) => ({ + id: c.id, + similarity: Number(c.similarity.toFixed(3)), + content: c.content.length > 800 ? `${c.content.slice(0, 800)}…` : c.content, + })); + return `You decide how to store a new auto-captured memory relative to similar existing memories. + +New memory draft: +${summary} + +Similar existing memories (JSON): +${JSON.stringify(items, null, 2)} + +Choose exactly one action: +- "add": new distinct fact; keep existing memories unchanged. +- "update": rewrite targetId in place with the corrected/current content (use "content"). +- "supersede": save the new memory and soft-delete targetId (outdated fact replaced). + +Rules: +- Prefer "add" when unsure or when memories are only loosely related. +- Use "update" / "supersede" only when the new draft clearly updates or replaces the same fact. +- targetId MUST be one of the candidate ids when action is update or supersede. +- For "update", "content" MUST retain important details (paths, commands, why something failed). +- Do NOT invent facts. Do NOT touch pinned memories (already excluded). +- Return structured JSON only.`; +} + +/** + * Apply an LLM decision. Invalid update/supersede targets fall back to add. + * Returns the persisted memory id (new or updated). + */ +export async function applyUpdateOnCaptureDecision( + client: UpdateOnCaptureMemoryClient, + decision: UpdateOnCaptureDecision, + input: PersistCaptureInput, + candidateIds: Set +): Promise<{ success: boolean; id?: string; action: UpdateOnCaptureAction; error?: string }> { + const fallbackAdd = async () => { + const result = await client.addMemory(input.summary, input.containerTag, input.metadata as any); + if (!result.success) { + return { success: false as const, action: "add" as const, error: result.error }; + } + return { success: true as const, id: result.id, action: "add" as const }; + }; + + const action = decision.action || "add"; + if (action === "add") { + return fallbackAdd(); + } + + const targetId = decision.targetId; + if (!targetId || !candidateIds.has(targetId)) { + log("update-on-capture: invalid targetId, falling back to add", { + action, + targetId, + }); + return fallbackAdd(); + } + + const existing = await client.getMemoryById(targetId); + if (!existing.success || !existing.memory || existing.memory.isPinned) { + log("update-on-capture: target missing or pinned, falling back to add", { + action, + targetId, + }); + return fallbackAdd(); + } + + if (action === "update") { + const content = (decision.content || input.summary).trim(); + if (!content) return fallbackAdd(); + const updated = await client.updateMemory(targetId, { + content, + type: input.type as MemoryType, + tags: input.tags, + metadataPatch: { + source: "auto-capture", + updatedByCapture: true, + promptId: input.metadata.promptId, + sessionID: input.metadata.sessionID, + captureTimestamp: input.metadata.captureTimestamp, + }, + }); + if (!updated.success) { + return { success: false, action: "update", error: updated.error }; + } + return { success: true, id: targetId, action: "update" }; + } + + // supersede + const added = await client.addMemory(input.summary, input.containerTag, input.metadata as any); + if (!added.success) { + return { success: false, action: "supersede", error: added.error }; + } + const deleted = await client.deleteMemory(targetId, { supersededBy: added.id }); + if (!deleted.success) { + log("update-on-capture: supersede soft-delete failed", { + targetId, + newId: added.id, + error: deleted.error, + }); + } + return { success: true, id: added.id, action: "supersede" }; +} diff --git a/tests/add-memory-phase-logging.test.ts b/tests/add-memory-phase-logging.test.ts index 52d84695..f600c05c 100644 --- a/tests/add-memory-phase-logging.test.ts +++ b/tests/add-memory-phase-logging.test.ts @@ -60,6 +60,7 @@ mock.module(${JSON.stringify(embeddingUrl)}, () => ({ mock.module(${JSON.stringify(vectorUtilsUrl)}, () => ({ formatTagsForEmbedding: (tags) => tags.join(" "), + parseSessionIdFromMetadata: () => undefined, })); mock.module(${JSON.stringify(shardManagerUrl)}, () => ({ diff --git a/tests/config.test.ts b/tests/config.test.ts index 9e7beceb..4ea67fda 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -118,6 +118,14 @@ describe("config", () => { expect(CONFIG.dreaming.intervalHours).toBe(24); expect(CONFIG.dreaming.maxGroupsPerRun).toBe(5); expect(CONFIG.dreaming.trashRetentionDays).toBe(30); + expect(CONFIG.dreaming.relatedThreshold).toBe(0.75); + expect(CONFIG.dreaming.crossProject).toBe(false); + }); + + it("should default update-on-capture to disabled", () => { + expect(CONFIG.updateOnCaptureEnabled).toBe(false); + expect(CONFIG.updateOnCaptureSimilarityThreshold).toBe(0.75); + expect(CONFIG.updateOnCaptureMaxCandidates).toBe(5); }); it("should have boolean toggle settings", () => { diff --git a/tests/update-on-capture.test.ts b/tests/update-on-capture.test.ts new file mode 100644 index 00000000..8f5f9c3f --- /dev/null +++ b/tests/update-on-capture.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "bun:test"; +import { + normalizeSimilarityUnitInterval, + normalizeUpdateOnCaptureMaxCandidates, +} from "../src/config.js"; +import { + applyUpdateOnCaptureDecision, + filterUpdateOnCaptureCandidates, + type UpdateOnCaptureMemoryClient, +} from "../src/services/update-on-capture.js"; + +describe("update-on-capture config helpers", () => { + it("clamps similarity thresholds to [0,1]", () => { + expect(normalizeSimilarityUnitInterval(0.75, 0.5)).toBe(0.75); + expect(normalizeSimilarityUnitInterval(-1, 0.5)).toBe(0); + expect(normalizeSimilarityUnitInterval(2, 0.5)).toBe(1); + expect(normalizeSimilarityUnitInterval("x", 0.5)).toBe(0.5); + }); + + it("normalizes max candidates", () => { + expect(normalizeUpdateOnCaptureMaxCandidates(3)).toBe(3); + expect(normalizeUpdateOnCaptureMaxCandidates(0)).toBe(5); + expect(normalizeUpdateOnCaptureMaxCandidates(99)).toBe(20); + }); +}); + +describe("filterUpdateOnCaptureCandidates", () => { + it("drops pinned and respects max", () => { + const out = filterUpdateOnCaptureCandidates( + [ + { id: "a", content: "a", similarity: 0.9, isPinned: true }, + { id: "b", content: "b", similarity: 0.8 }, + { id: "c", content: "c", similarity: 0.85 }, + { id: "d", content: "d", similarity: 0.7 }, + ], + 2 + ); + expect(out.map((c) => c.id)).toEqual(["c", "b"]); + }); +}); + +describe("applyUpdateOnCaptureDecision", () => { + const baseInput = { + summary: "KyuShared is on sda5", + type: "configuration", + tags: ["linux"], + containerTag: "project_abc", + metadata: { + source: "auto-capture" as const, + type: "configuration", + tags: ["linux"], + sessionID: "s1", + promptId: "p1", + captureTimestamp: 1, + }, + }; + + function mockClient( + overrides: Partial = {} + ): UpdateOnCaptureMemoryClient & { + calls: string[]; + } { + const calls: string[] = []; + return { + calls, + addMemory: async () => { + calls.push("add"); + return { success: true, id: "mem_new" }; + }, + updateMemory: async (id) => { + calls.push(`update:${id}`); + return { success: true }; + }, + deleteMemory: async (id, patch) => { + calls.push(`delete:${id}:${JSON.stringify(patch)}`); + return { success: true }; + }, + getMemoryById: async (id) => { + calls.push(`get:${id}`); + return { success: true, memory: { id, content: "old", isPinned: false } }; + }, + ...overrides, + }; + } + + it("adds when action is add", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "add" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result).toEqual({ success: true, id: "mem_new", action: "add" }); + expect(client.calls).toEqual(["add"]); + }); + + it("updates a valid candidate", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "update", targetId: "mem_old", content: "rewritten" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.action).toBe("update"); + expect(result.id).toBe("mem_old"); + expect(client.calls[0]).toBe("get:mem_old"); + expect(client.calls[1]).toBe("update:mem_old"); + }); + + it("supersedes by adding then soft-deleting", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "supersede", targetId: "mem_old" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result).toEqual({ success: true, id: "mem_new", action: "supersede" }); + expect(client.calls).toContain("add"); + expect(client.calls.some((c) => c.startsWith("delete:mem_old:"))).toBe(true); + }); + + it("falls back to add for invalid target", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "update", targetId: "missing" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.action).toBe("add"); + expect(client.calls).toEqual(["add"]); + }); + + it("falls back to add when target is pinned", async () => { + const client = mockClient({ + getMemoryById: async (id) => ({ + success: true, + memory: { id, content: "old", isPinned: true }, + }), + }); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "update", targetId: "mem_old", content: "x" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.action).toBe("add"); + }); +}); From b3be861417d5dd39060237d49ce3f46bdc3a0645 Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:56:22 +0100 Subject: [PATCH 2/6] feat(dreaming): related-threshold consolidation with supersede and flag Use dreaming.relatedThreshold for grouping, support merge/update/supersede/ keep/flag actions, and allow cross-project groups when configured. --- src/services/dreaming-service.ts | 226 ++++++++++++++++++++++------ src/services/turso/vector-search.ts | 17 +++ tests/dreaming-prompt.test.ts | 35 +++++ 3 files changed, 228 insertions(+), 50 deletions(-) diff --git a/src/services/dreaming-service.ts b/src/services/dreaming-service.ts index 04b6e4e2..dd68fd32 100644 --- a/src/services/dreaming-service.ts +++ b/src/services/dreaming-service.ts @@ -13,6 +13,8 @@ import { parseExtractedVector, formatTagsForEmbedding } from "./turso/vector-uti import { getV2Client } from "./ai/opencode-provider.js"; import { tryAcquireDreamingLock } from "./dreaming-lock.js"; +const MAX_GROUP_SIZE = 5; + export interface DreamingNearDupGroup { representative: { id: string; @@ -20,12 +22,15 @@ export interface DreamingNearDupGroup { containerTag: string; createdAt: number; isPinned: boolean; + scope: "user" | "project"; }; duplicates: Array<{ id: string; content: string; similarity: number; isPinned: boolean; + containerTag: string; + createdAt: number; }>; } @@ -33,6 +38,7 @@ export interface DreamingRunResult { groupsConsidered: number; groupsMerged: number; groupsSkipped: number; + groupsFlagged: number; memoriesSoftDeleted: number; } @@ -40,8 +46,8 @@ interface DreamingState { lastRunAt: number; } -const mergeSchema = z.object({ - action: z.enum(["merge", "skip"]), +const dreamingActionSchema = z.object({ + action: z.enum(["merge", "update", "supersede", "keep", "flag"]), content: z.string().optional(), keepId: z.string().optional(), dropIds: z.array(z.string()).optional(), @@ -69,79 +75,146 @@ function writeState(state: DreamingState): void { writeFileSync(statePath(), JSON.stringify(state), "utf-8"); } +export function isDreamingCrossProjectEnabled(): boolean { + return Boolean(CONFIG.dreaming.crossProject) || CONFIG.memory.defaultScope === "all-projects"; +} + +type MemoryRow = { + id: unknown; + content: unknown; + container_tag: unknown; + created_at: unknown; + is_pinned?: unknown; + vector_json?: unknown; +}; + +/** + * Group related memories for dreaming. Uses dreaming.relatedThreshold + * (not deduplicationSimilarityThreshold). Cross-project grouping is opt-in / + * follows memory.defaultScope; user-scope never mixes with project-scope. + */ export async function findNearDuplicateGroupsForDreaming(): Promise { await ensureTursoReady(); const userShards = await tursoShardManager.getAllShards("user", ""); const projectShards = await tursoShardManager.getAllShards("project", ""); - const allShards = [...userShards, ...projectShards]; + const threshold = CONFIG.dreaming.relatedThreshold; + const crossProject = isDreamingCrossProjectEnabled(); const groups: DreamingNearDupGroup[] = []; - const threshold = CONFIG.deduplicationSimilarityThreshold; - for (const shard of allShards) { - const db = await tursoConnectionManager.getConnection(shard.dbPath); - const memories = await tursoVectorSearch.getAllMemoriesWithExtractedVectors(db); - const processedIds = new Set(); + const collectFromShardPool = async ( + shards: Awaited>, + scope: "user" | "project", + allowCrossTag: boolean + ) => { + type Entry = { + id: string; + content: string; + containerTag: string; + createdAt: number; + vector: Float32Array; + shardDbPath: string; + }; + const entries: Entry[] = []; - for (let i = 0; i < memories.length; i++) { - const mem1 = memories[i]!; - if (Number(mem1.is_pinned ?? 0) === 1) continue; - const vector1 = parseExtractedVector(mem1.vector_json); - if (!vector1 || processedIds.has(String(mem1.id))) continue; + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.getAllMemoriesWithExtractedVectors(db); + for (const mem of memories as MemoryRow[]) { + if (Number(mem.is_pinned ?? 0) === 1) continue; + const vector = parseExtractedVector(mem.vector_json); + if (!vector) continue; + entries.push({ + id: String(mem.id), + content: String(mem.content), + containerTag: String(mem.container_tag), + createdAt: Number(mem.created_at), + vector, + shardDbPath: shard.dbPath, + }); + } + } + + const processedIds = new Set(); + for (let i = 0; i < entries.length; i++) { + const mem1 = entries[i]!; + if (processedIds.has(mem1.id)) continue; const similarGroup: DreamingNearDupGroup = { representative: { - id: String(mem1.id), - content: String(mem1.content), - containerTag: String(mem1.container_tag), - createdAt: Number(mem1.created_at), + id: mem1.id, + content: mem1.content, + containerTag: mem1.containerTag, + createdAt: mem1.createdAt, isPinned: false, + scope, }, duplicates: [], }; - for (let j = i + 1; j < memories.length; j++) { - const mem2 = memories[j]!; - if (Number(mem2.is_pinned ?? 0) === 1) continue; - const vector2 = parseExtractedVector(mem2.vector_json); - if (!vector2 || processedIds.has(String(mem2.id))) continue; - if (mem1.container_tag !== mem2.container_tag) continue; + for (let j = i + 1; j < entries.length; j++) { + const mem2 = entries[j]!; + if (processedIds.has(mem2.id)) continue; + if (!allowCrossTag && mem1.containerTag !== mem2.containerTag) continue; - const similarity = cosineSimilarity(vector1, vector2); + const similarity = cosineSimilarity(mem1.vector, mem2.vector); if (similarity >= threshold && similarity < 1.0) { similarGroup.duplicates.push({ - id: String(mem2.id), - content: String(mem2.content), + id: mem2.id, + content: mem2.content, similarity, isPinned: false, + containerTag: mem2.containerTag, + createdAt: mem2.createdAt, }); - processedIds.add(String(mem2.id)); + processedIds.add(mem2.id); + if (similarGroup.duplicates.length >= MAX_GROUP_SIZE - 1) break; } } if (similarGroup.duplicates.length > 0) { - processedIds.add(String(mem1.id)); + processedIds.add(mem1.id); groups.push(similarGroup); } } - } + }; + + // User-scope: always same container_tag only (never mix with projects). + await collectFromShardPool(userShards, "user", false); + // Project-scope: cross-tag when crossProject / defaultScope all-projects. + await collectFromShardPool(projectShards, "project", crossProject); return groups; } -function buildMergePrompt(group: DreamingNearDupGroup): string { +export function buildMergePrompt(group: DreamingNearDupGroup): string { const items = [ - { id: group.representative.id, content: group.representative.content }, - ...group.duplicates.map((d) => ({ id: d.id, content: d.content })), + { + id: group.representative.id, + content: group.representative.content, + createdAt: group.representative.createdAt, + containerTag: group.representative.containerTag, + }, + ...group.duplicates.map((d) => ({ + id: d.id, + content: d.content, + createdAt: d.createdAt, + containerTag: d.containerTag, + similarity: Number(d.similarity.toFixed(3)), + })), ]; - return `You consolidate near-duplicate project memories into one clearer entry. + return `You consolidate related project/user memories. They may be near-duplicates OR newer facts that replace older ones. Rules: -- Prefer action "merge" only when the memories clearly describe the same fact/fix/gotcha. -- The merged content MUST retain important details, including why something failed when present. +- "merge": same fact worded differently → one clearer entry (keepId + dropIds + content). +- "update": rewrite keepId with the current correct content; soft-delete dropIds. +- "supersede": newer fact replaces older → keep newest correct content on keepId (or prefer newest id), soft-delete outdated dropIds. +- "keep": related but all still valid / unsafe to change — no mutation. +- "flag": clear contradiction you cannot resolve — no mutation; we mark metadata. +- Prefer the newest correct fact when one clearly supersedes another. +- Merged/updated content MUST retain important details, including why something failed when present. - Do NOT invent facts. Do NOT drop unique constraints, commands, paths, or error messages. -- keepId must be one of the input ids (prefer the most complete/recent). -- dropIds must list every other id when merging. -- Use action "skip" when memories are related but not safe to merge. +- keepId must be one of the input ids. dropIds lists every other id when merging/updating/superseding. +- Use "keep" when memories are only loosely related. Return JSON matching the schema. @@ -159,6 +232,9 @@ export class DreamingService { intervalHours: CONFIG.dreaming.intervalHours, maxGroupsPerRun: CONFIG.dreaming.maxGroupsPerRun, trashRetentionDays: CONFIG.dreaming.trashRetentionDays, + relatedThreshold: CONFIG.dreaming.relatedThreshold, + crossProject: CONFIG.dreaming.crossProject, + crossProjectEffective: isDreamingCrossProjectEnabled(), isRunning: this.isRunning, lastRunAt: state.lastRunAt || null, }; @@ -201,6 +277,7 @@ export class DreamingService { groupsConsidered: 0, groupsMerged: 0, groupsSkipped: 0, + groupsFlagged: 0, memoriesSoftDeleted: 0, }; @@ -233,19 +310,37 @@ export class DreamingService { systemPrompt: "You are a careful memory consolidation assistant. Return only structured JSON.", userPrompt: buildMergePrompt(group), - schema: mergeSchema, + schema: dreamingActionSchema, directory, }); - if (decision.action !== "merge" || !decision.content?.trim()) { - result.groupsSkipped++; - continue; - } - const memberIds = new Set([ group.representative.id, ...group.duplicates.map((d) => d.id), ]); + + if (decision.action === "keep") { + result.groupsSkipped++; + continue; + } + + if (decision.action === "flag") { + const flagged = await this.applyFlag(group, decision.reason); + if (flagged) result.groupsFlagged++; + else result.groupsSkipped++; + continue; + } + + if ( + (decision.action !== "merge" && + decision.action !== "update" && + decision.action !== "supersede") || + !decision.content?.trim() + ) { + result.groupsSkipped++; + continue; + } + const keepId = decision.keepId && memberIds.has(decision.keepId) ? decision.keepId @@ -261,11 +356,14 @@ export class DreamingService { continue; } - const applied = await this.applyMerge( + const metaKey = decision.action === "supersede" ? "supersededBy" : "mergedInto"; + const applied = await this.applyDreamingAction( keepId, decision.content.trim(), finalDrop, - group.representative.containerTag + group.representative.containerTag, + metaKey, + decision.action ); if (applied) { result.groupsMerged++; @@ -288,11 +386,37 @@ export class DreamingService { } } - private async applyMerge( + private async applyFlag(group: DreamingNearDupGroup, reason?: string): Promise { + const ids = [group.representative.id, ...group.duplicates.map((d) => d.id)]; + const shards = [ + ...(await tursoShardManager.getAllShards("user", "")), + ...(await tursoShardManager.getAllShards("project", "")), + ]; + let any = false; + for (const id of ids) { + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const row = await tursoVectorSearch.getMemoryById(db, id); + if (!row) continue; + const ok = await tursoVectorSearch.patchMemoryMetadata(db, id, { + contradictionWith: ids.filter((other) => other !== id), + contradictionReason: reason || "Unresolved contradiction during dreaming", + contradictionFlaggedAt: Date.now(), + }); + if (ok) any = true; + break; + } + } + return any; + } + + private async applyDreamingAction( keepId: string, content: string, dropIds: string[], - containerTag: string + containerTag: string, + dropMetaKey: "mergedInto" | "supersededBy", + action: "merge" | "update" | "supersede" ): Promise { const shards = [ ...(await tursoShardManager.getAllShards("user", "")), @@ -339,10 +463,12 @@ export class DreamingService { } meta.mergedFrom = mergedFrom; meta.source = typeof meta.source === "string" ? meta.source : "dreaming"; + meta.dreamingAction = action; const db = await tursoConnectionManager.getConnection(keepShard.dbPath); const updatedAt = Date.now(); const isPinned = Number(keepRow.is_pinned ?? 0) === 1 ? 1 : 0; + const keepContainerTag = containerTag || String(keepRow.container_tag); await db.transaction("write", async (tx) => { await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [keepId] }); @@ -359,7 +485,7 @@ export class DreamingService { content, JSON.stringify(Array.from(vector)), ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), - containerTag || String(keepRow.container_tag), + keepContainerTag, tags.length > 0 ? tags.join(",") : null, keepRow.type ? String(keepRow.type) : null, Number(keepRow.created_at), @@ -383,7 +509,7 @@ export class DreamingService { const dropRow = await tursoVectorSearch.getMemoryById(dropDb, dropId); if (!dropRow) continue; if (Number(dropRow.is_pinned ?? 0) === 1) continue; - await tursoVectorSearch.softDeleteVector(dropDb, dropId, { mergedInto: keepId }); + await tursoVectorSearch.softDeleteVector(dropDb, dropId, { [dropMetaKey]: keepId }); break; } } diff --git a/src/services/turso/vector-search.ts b/src/services/turso/vector-search.ts index f6249d2e..76fdf033 100644 --- a/src/services/turso/vector-search.ts +++ b/src/services/turso/vector-search.ts @@ -361,6 +361,23 @@ export class TursoVectorSearch { return changes > 0; } + async patchMemoryMetadata( + db: TursoDb, + memoryId: string, + metaPatch: Record + ): Promise { + await this.prepareShardDb(db); + const existing = await this.getMemoryById(db, memoryId, { includeDeleted: false }); + if (!existing) return false; + const meta = parseMetadata(existing.metadata) || {}; + Object.assign(meta, metaPatch); + const changes = await db.run( + `UPDATE memories SET metadata = ?, updated_at = ? WHERE id = ? AND ${ACTIVE_MEMORY_SQL}`, + [JSON.stringify(meta), Date.now(), memoryId] + ); + return changes > 0; + } + async restoreVector(db: TursoDb, memoryId: string): Promise { await this.prepareShardDb(db); const changes = await db.run( diff --git a/tests/dreaming-prompt.test.ts b/tests/dreaming-prompt.test.ts index c06226b6..246222d0 100644 --- a/tests/dreaming-prompt.test.ts +++ b/tests/dreaming-prompt.test.ts @@ -2,9 +2,11 @@ import { describe, expect, it } from "bun:test"; import { normalizeDreamingIntervalHours, normalizeDreamingMaxGroupsPerRun, + normalizeDreamingRelatedThreshold, normalizeDreamingTrashRetentionDays, } from "../src/config.js"; import { mergeBoth, applyProjectBoost } from "../src/services/memory-recall.js"; +import { buildMergePrompt, type DreamingNearDupGroup } from "../src/services/dreaming-service.js"; describe("dreaming config helpers", () => { it("normalizes dreaming numeric knobs", () => { @@ -13,6 +15,39 @@ describe("dreaming config helpers", () => { expect(normalizeDreamingMaxGroupsPerRun(2.9)).toBe(2); expect(normalizeDreamingTrashRetentionDays(7)).toBe(7); expect(normalizeDreamingTrashRetentionDays(-1)).toBe(30); + expect(normalizeDreamingRelatedThreshold(0.8)).toBe(0.8); + expect(normalizeDreamingRelatedThreshold(2)).toBe(1); + expect(normalizeDreamingRelatedThreshold(undefined)).toBe(0.75); + }); +}); + +describe("dreaming prompt actions", () => { + it("mentions supersede/update/flag actions", () => { + const group: DreamingNearDupGroup = { + representative: { + id: "a", + content: "old partition nvme0n1p3", + containerTag: "p1", + createdAt: 1, + isPinned: false, + scope: "project", + }, + duplicates: [ + { + id: "b", + content: "moved to sda5", + similarity: 0.78, + isPinned: false, + containerTag: "p1", + createdAt: 2, + }, + ], + }; + const prompt = buildMergePrompt(group); + expect(prompt).toContain("supersede"); + expect(prompt).toContain("update"); + expect(prompt).toContain("flag"); + expect(prompt).toContain("keep"); }); }); From 913740ef71c040aba3d9891a8f1df21b2ae95169 Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:57:56 +0100 Subject: [PATCH 3/6] docs: keep PR1 dreaming README accurate until deeper dreaming lands --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5d2500af..e4b92b34 100644 --- a/README.md +++ b/README.md @@ -300,15 +300,15 @@ memories. Prefixes stay opt-in by default. ### Dreaming (optional memory consolidation) -Opt-in background pass that finds **related** memories (via `dreaming.relatedThreshold`, -default `0.75`, independent of `deduplicationSimilarityThreshold`), asks the configured -`opencodeProvider` / `opencodeModel` to `merge`, `update`, `supersede`, `keep`, or `flag` -contradictions, and soft-deletes superseded rows into recoverable trash. Pinned memories -are never touched. Set `dreaming.crossProject: true` (or `memory.defaultScope: -"all-projects"`) to group across project tags; user-scope stays separate. Configure under -`dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). The web UI exposes -**Trash** (restore) and **Dream** (manual run). Soft-deleted rows are hard-purged after -`dreaming.trashRetentionDays` during normal cleanup. +Opt-in background pass that finds near-duplicate memories (same similarity threshold as +deduplication), asks the configured `opencodeProvider` / `opencodeModel` to merge them, +and soft-deletes superseded rows into recoverable trash. Pinned memories are never +touched. Configure under `dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). +The web UI exposes **Trash** (restore) and **Dream** (manual run). Soft-deleted rows are +hard-purged after `dreaming.trashRetentionDays` during normal cleanup. + +Deeper related-threshold dreaming (update/supersede/flag, cross-project) lands in the +follow-up PR for #464. ### Choosing / configuring embeddings From 04ed4a10758af0f27a51916ed3ad378b7f4fa177 Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:58:22 +0100 Subject: [PATCH 4/6] fix(dreaming): preserve keep container_tag and cover related grouping Avoid rewriting keep rows onto the representative project tag in cross-project groups. Extract pure grouping for threshold/cross-tag tests and harden pinned-candidate filtering on capture. --- README.md | 18 ++-- src/services/dreaming-service.ts | 141 ++++++++++++++++++------------ src/services/update-on-capture.ts | 2 +- tests/dreaming-grouping.test.ts | 108 +++++++++++++++++++++++ 4 files changed, 203 insertions(+), 66 deletions(-) create mode 100644 tests/dreaming-grouping.test.ts diff --git a/README.md b/README.md index e4b92b34..5d2500af 100644 --- a/README.md +++ b/README.md @@ -300,15 +300,15 @@ memories. Prefixes stay opt-in by default. ### Dreaming (optional memory consolidation) -Opt-in background pass that finds near-duplicate memories (same similarity threshold as -deduplication), asks the configured `opencodeProvider` / `opencodeModel` to merge them, -and soft-deletes superseded rows into recoverable trash. Pinned memories are never -touched. Configure under `dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). -The web UI exposes **Trash** (restore) and **Dream** (manual run). Soft-deleted rows are -hard-purged after `dreaming.trashRetentionDays` during normal cleanup. - -Deeper related-threshold dreaming (update/supersede/flag, cross-project) lands in the -follow-up PR for #464. +Opt-in background pass that finds **related** memories (via `dreaming.relatedThreshold`, +default `0.75`, independent of `deduplicationSimilarityThreshold`), asks the configured +`opencodeProvider` / `opencodeModel` to `merge`, `update`, `supersede`, `keep`, or `flag` +contradictions, and soft-deletes superseded rows into recoverable trash. Pinned memories +are never touched. Set `dreaming.crossProject: true` (or `memory.defaultScope: +"all-projects"`) to group across project tags; user-scope stays separate. Configure under +`dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). The web UI exposes +**Trash** (restore) and **Dream** (manual run). Soft-deleted rows are hard-purged after +`dreaming.trashRetentionDays` during normal cleanup. ### Choosing / configuring embeddings diff --git a/src/services/dreaming-service.ts b/src/services/dreaming-service.ts index dd68fd32..185c8b35 100644 --- a/src/services/dreaming-service.ts +++ b/src/services/dreaming-service.ts @@ -13,7 +13,7 @@ import { parseExtractedVector, formatTagsForEmbedding } from "./turso/vector-uti import { getV2Client } from "./ai/opencode-provider.js"; import { tryAcquireDreamingLock } from "./dreaming-lock.js"; -const MAX_GROUP_SIZE = 5; +export const DREAMING_MAX_GROUP_SIZE = 5; export interface DreamingNearDupGroup { representative: { @@ -34,6 +34,78 @@ export interface DreamingNearDupGroup { }>; } +export type DreamingMemoryEntry = { + id: string; + content: string; + containerTag: string; + createdAt: number; + vector: Float32Array; +}; + +/** + * Pure grouping used by dreaming (and unit tests). Pinned rows must be + * excluded by the caller before invoking this. + */ +export function groupRelatedMemoryEntries( + entries: DreamingMemoryEntry[], + options: { + threshold: number; + allowCrossTag: boolean; + scope: "user" | "project"; + maxGroupSize?: number; + } +): DreamingNearDupGroup[] { + const threshold = options.threshold; + const allowCrossTag = options.allowCrossTag; + const maxGroupSize = options.maxGroupSize ?? DREAMING_MAX_GROUP_SIZE; + const groups: DreamingNearDupGroup[] = []; + const processedIds = new Set(); + + for (let i = 0; i < entries.length; i++) { + const mem1 = entries[i]!; + if (processedIds.has(mem1.id)) continue; + + const similarGroup: DreamingNearDupGroup = { + representative: { + id: mem1.id, + content: mem1.content, + containerTag: mem1.containerTag, + createdAt: mem1.createdAt, + isPinned: false, + scope: options.scope, + }, + duplicates: [], + }; + + for (let j = i + 1; j < entries.length; j++) { + const mem2 = entries[j]!; + if (processedIds.has(mem2.id)) continue; + if (!allowCrossTag && mem1.containerTag !== mem2.containerTag) continue; + + const similarity = cosineSimilarity(mem1.vector, mem2.vector); + if (similarity >= threshold && similarity < 1.0) { + similarGroup.duplicates.push({ + id: mem2.id, + content: mem2.content, + similarity, + isPinned: false, + containerTag: mem2.containerTag, + createdAt: mem2.createdAt, + }); + processedIds.add(mem2.id); + if (similarGroup.duplicates.length >= maxGroupSize - 1) break; + } + } + + if (similarGroup.duplicates.length > 0) { + processedIds.add(mem1.id); + groups.push(similarGroup); + } + } + + return groups; +} + export interface DreamingRunResult { groupsConsidered: number; groupsMerged: number; @@ -106,15 +178,7 @@ export async function findNearDuplicateGroupsForDreaming(): Promise { - type Entry = { - id: string; - content: string; - containerTag: string; - createdAt: number; - vector: Float32Array; - shardDbPath: string; - }; - const entries: Entry[] = []; + const entries: DreamingMemoryEntry[] = []; for (const shard of shards) { const db = await tursoConnectionManager.getConnection(shard.dbPath); @@ -129,53 +193,18 @@ export async function findNearDuplicateGroupsForDreaming(): Promise(); - for (let i = 0; i < entries.length; i++) { - const mem1 = entries[i]!; - if (processedIds.has(mem1.id)) continue; - - const similarGroup: DreamingNearDupGroup = { - representative: { - id: mem1.id, - content: mem1.content, - containerTag: mem1.containerTag, - createdAt: mem1.createdAt, - isPinned: false, - scope, - }, - duplicates: [], - }; - - for (let j = i + 1; j < entries.length; j++) { - const mem2 = entries[j]!; - if (processedIds.has(mem2.id)) continue; - if (!allowCrossTag && mem1.containerTag !== mem2.containerTag) continue; - - const similarity = cosineSimilarity(mem1.vector, mem2.vector); - if (similarity >= threshold && similarity < 1.0) { - similarGroup.duplicates.push({ - id: mem2.id, - content: mem2.content, - similarity, - isPinned: false, - containerTag: mem2.containerTag, - createdAt: mem2.createdAt, - }); - processedIds.add(mem2.id); - if (similarGroup.duplicates.length >= MAX_GROUP_SIZE - 1) break; - } - } - - if (similarGroup.duplicates.length > 0) { - processedIds.add(mem1.id); - groups.push(similarGroup); - } - } + groups.push( + ...groupRelatedMemoryEntries(entries, { + threshold, + allowCrossTag, + scope, + maxGroupSize: DREAMING_MAX_GROUP_SIZE, + }) + ); }; // User-scope: always same container_tag only (never mix with projects). @@ -357,11 +386,12 @@ export class DreamingService { } const metaKey = decision.action === "supersede" ? "supersededBy" : "mergedInto"; + // Always preserve the keep row's own container_tag (critical for + // cross-project groups where keepId may not be the representative). const applied = await this.applyDreamingAction( keepId, decision.content.trim(), finalDrop, - group.representative.containerTag, metaKey, decision.action ); @@ -414,7 +444,6 @@ export class DreamingService { keepId: string, content: string, dropIds: string[], - containerTag: string, dropMetaKey: "mergedInto" | "supersededBy", action: "merge" | "update" | "supersede" ): Promise { @@ -468,7 +497,7 @@ export class DreamingService { const db = await tursoConnectionManager.getConnection(keepShard.dbPath); const updatedAt = Date.now(); const isPinned = Number(keepRow.is_pinned ?? 0) === 1 ? 1 : 0; - const keepContainerTag = containerTag || String(keepRow.container_tag); + const keepContainerTag = String(keepRow.container_tag); await db.transaction("write", async (tx) => { await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [keepId] }); diff --git a/src/services/update-on-capture.ts b/src/services/update-on-capture.ts index ee8d0489..de71683a 100644 --- a/src/services/update-on-capture.ts +++ b/src/services/update-on-capture.ts @@ -71,7 +71,7 @@ export function filterUpdateOnCaptureCandidates( maxCandidates = CONFIG.updateOnCaptureMaxCandidates ): UpdateOnCaptureCandidate[] { return candidates - .filter((c) => !c.isPinned) + .filter((c) => Number(c.isPinned) !== 1 && c.isPinned !== true) .sort((a, b) => b.similarity - a.similarity) .slice(0, Math.max(1, maxCandidates)); } diff --git a/tests/dreaming-grouping.test.ts b/tests/dreaming-grouping.test.ts new file mode 100644 index 00000000..617b0174 --- /dev/null +++ b/tests/dreaming-grouping.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "bun:test"; +import { + groupRelatedMemoryEntries, + isDreamingCrossProjectEnabled, + type DreamingMemoryEntry, +} from "../src/services/dreaming-service.js"; +import { CONFIG } from "../src/config.js"; + +function vec(...values: number[]): Float32Array { + return new Float32Array(values); +} + +function entry( + id: string, + containerTag: string, + vector: Float32Array, + createdAt = 1 +): DreamingMemoryEntry { + return { id, content: id, containerTag, createdAt, vector }; +} + +describe("groupRelatedMemoryEntries", () => { + it("groups related pairs at relatedThreshold but not only near-dup 0.9", () => { + // Identical direction → similarity 1.0 is excluded (< 1.0 required). + // Nearly parallel vectors → high similarity (~0.99). + // Distinct-ish vectors → ~0.8 range. + const a = entry("a", "p1", vec(1, 0, 0)); + const b = entry("b", "p1", vec(0.8, 0.6, 0)); // cos ≈ 0.8 + const c = entry("c", "p1", vec(0, 1, 0)); // cos with a = 0 + + const at075 = groupRelatedMemoryEntries([a, b, c], { + threshold: 0.75, + allowCrossTag: false, + scope: "project", + }); + expect(at075).toHaveLength(1); + expect(at075[0]!.representative.id).toBe("a"); + expect(at075[0]!.duplicates.map((d) => d.id)).toEqual(["b"]); + + const at09 = groupRelatedMemoryEntries([a, b, c], { + threshold: 0.9, + allowCrossTag: false, + scope: "project", + }); + expect(at09).toHaveLength(0); + }); + + it("keeps same-tag only when cross-tag is disabled", () => { + const a = entry("a", "p1", vec(1, 0)); + const b = entry("b", "p2", vec(0.99, 0.01)); + const groups = groupRelatedMemoryEntries([a, b], { + threshold: 0.75, + allowCrossTag: false, + scope: "project", + }); + expect(groups).toHaveLength(0); + }); + + it("allows cross-tag grouping when enabled", () => { + const a = entry("a", "p1", vec(1, 0)); + const b = entry("b", "p2", vec(0.99, 0.01)); + const groups = groupRelatedMemoryEntries([a, b], { + threshold: 0.75, + allowCrossTag: true, + scope: "project", + }); + expect(groups).toHaveLength(1); + expect(groups[0]!.duplicates[0]!.containerTag).toBe("p2"); + }); + + it("caps group size", () => { + const base = entry("r", "p1", vec(1, 0, 0)); + const others = [0.1, 0.15, 0.2, 0.25, 0.3].map((y, i) => + entry(`d${i}`, "p1", vec(1, y, 0), i + 2) + ); + const groups = groupRelatedMemoryEntries([base, ...others], { + threshold: 0.7, + allowCrossTag: false, + scope: "project", + maxGroupSize: 3, + }); + expect(groups.length).toBeGreaterThanOrEqual(1); + expect(groups.every((g) => g.duplicates.length <= 2)).toBe(true); + expect(groups[0]!.duplicates.length).toBe(2); + }); +}); + +describe("isDreamingCrossProjectEnabled", () => { + it("follows dreaming.crossProject or memory.defaultScope", () => { + const prevCross = CONFIG.dreaming.crossProject; + const prevScope = CONFIG.memory.defaultScope; + try { + CONFIG.dreaming.crossProject = false; + CONFIG.memory.defaultScope = "project"; + expect(isDreamingCrossProjectEnabled()).toBe(false); + + CONFIG.memory.defaultScope = "all-projects"; + expect(isDreamingCrossProjectEnabled()).toBe(true); + + CONFIG.memory.defaultScope = "project"; + CONFIG.dreaming.crossProject = true; + expect(isDreamingCrossProjectEnabled()).toBe(true); + } finally { + CONFIG.dreaming.crossProject = prevCross; + CONFIG.memory.defaultScope = prevScope; + } + }); +}); From c6afb02bb5cbaa6f0066219b6b669f97e4cb6b69 Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:00:36 +0100 Subject: [PATCH 5/6] fix(memory): harden supersede trash path for #464 Surface supersededBy in trash/restore UI, roll back the new memory when soft-delete fails, and over-fetch capture candidates before pin filtering. --- src/services/api-handlers.ts | 13 +++++++- src/services/auto-capture.ts | 27 ++++++++++++++-- src/services/update-on-capture.ts | 13 ++++++-- tests/update-on-capture.test.ts | 32 ++++++++++++++++++- .../components/explorer/TrashDialog.svelte | 4 +++ web/src/lib/i18n/translations.ts | 6 ++++ .../lib/stores/memories-explorer.svelte.ts | 1 + 7 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 0ddc260a..7e462ff0 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -506,6 +506,7 @@ export async function handleListTrash(limit = 200): Promise< content: string; deletedAt: number; mergedInto?: string; + supersededBy?: string; projectName?: string; containerTag: string; }> @@ -518,6 +519,7 @@ export async function handleListTrash(limit = 200): Promise< content: string; deletedAt: number; mergedInto?: string; + supersededBy?: string; projectName?: string; containerTag: string; }> = []; @@ -531,6 +533,7 @@ export async function handleListTrash(limit = 200): Promise< content: String(row.content), deletedAt: Number(row.deleted_at), mergedInto: typeof meta.mergedInto === "string" ? meta.mergedInto : undefined, + supersededBy: typeof meta.supersededBy === "string" ? meta.supersededBy : undefined, projectName: row.project_name ? String(row.project_name) : undefined, containerTag: String(row.container_tag), }); @@ -554,10 +557,18 @@ export async function handleRestoreMemory(id: string): Promise = { + add: { + title: "Memory Captured", + message: "Project memory saved from conversation", + }, + update: { + title: "Memory Updated", + message: "Similar existing memory was rewritten with current facts", + }, + supersede: { + title: "Memory Superseded", + message: "New memory saved; outdated entry moved to trash", + }, + }; + const toast = toastByAction[action] ?? toastByAction.add!; await ctx.client?.tui .showToast({ body: { - title: "Memory Captured", - message: "Project memory saved from conversation", + title: toast.title, + message: toast.message, variant: "success", duration: 3000, }, @@ -752,9 +768,14 @@ async function persistAutoCaptureMemory( } const scope = CONFIG.memory.defaultScope === "all-projects" ? "all-projects" : "project"; + // Over-fetch before pin-filtering so pinned hits don't starve the candidate list. + const searchLimit = Math.min( + 20, + Math.max(CONFIG.updateOnCaptureMaxCandidates * 3, CONFIG.updateOnCaptureMaxCandidates + 5) + ); const search = await memoryClient.searchMemories(summaryWithTags, containerTag, scope, { similarityThreshold: CONFIG.updateOnCaptureSimilarityThreshold, - limit: CONFIG.updateOnCaptureMaxCandidates, + limit: searchLimit, }); const rawCandidates: UpdateOnCaptureCandidate[] = (search.success ? search.results : []).map( diff --git a/src/services/update-on-capture.ts b/src/services/update-on-capture.ts index de71683a..613d1915 100644 --- a/src/services/update-on-capture.ts +++ b/src/services/update-on-capture.ts @@ -171,16 +171,23 @@ export async function applyUpdateOnCaptureDecision( // supersede const added = await client.addMemory(input.summary, input.containerTag, input.metadata as any); - if (!added.success) { - return { success: false, action: "supersede", error: added.error }; + if (!added.success || !added.id) { + return { success: false, action: "supersede", error: added.error || "add failed" }; } const deleted = await client.deleteMemory(targetId, { supersededBy: added.id }); if (!deleted.success) { - log("update-on-capture: supersede soft-delete failed", { + log("update-on-capture: supersede soft-delete failed; rolling back new memory", { targetId, newId: added.id, error: deleted.error, }); + // Keep store consistent: don't leave both old (active) and new (active). + await client.deleteMemory(added.id, { supersededRollback: true }); + return { + success: false, + action: "supersede", + error: deleted.error || "failed to soft-delete superseded memory", + }; } return { success: true, id: added.id, action: "supersede" }; } diff --git a/tests/update-on-capture.test.ts b/tests/update-on-capture.test.ts index 8f5f9c3f..23b8c875 100644 --- a/tests/update-on-capture.test.ts +++ b/tests/update-on-capture.test.ts @@ -119,7 +119,37 @@ describe("applyUpdateOnCaptureDecision", () => { ); expect(result).toEqual({ success: true, id: "mem_new", action: "supersede" }); expect(client.calls).toContain("add"); - expect(client.calls.some((c) => c.startsWith("delete:mem_old:"))).toBe(true); + expect(client.calls.some((c) => c.includes("supersededBy"))).toBe(true); + }); + + it("rolls back new memory when supersede soft-delete fails", async () => { + const calls: string[] = []; + const client: UpdateOnCaptureMemoryClient & { calls: string[] } = { + calls, + addMemory: async () => { + calls.push("add"); + return { success: true, id: "mem_new" }; + }, + updateMemory: async () => ({ success: true }), + getMemoryById: async (id) => ({ + success: true, + memory: { id, content: "old", isPinned: false }, + }), + deleteMemory: async (id, patch) => { + calls.push(`delete:${id}:${JSON.stringify(patch)}`); + if (id === "mem_old") return { success: false, error: "busy" }; + return { success: true }; + }, + }; + const result = await applyUpdateOnCaptureDecision( + client, + { action: "supersede", targetId: "mem_old" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.success).toBe(false); + expect(result.action).toBe("supersede"); + expect(calls.some((c) => c.includes("supersededRollback"))).toBe(true); }); it("falls back to add for invalid target", async () => { diff --git a/web/src/lib/components/explorer/TrashDialog.svelte b/web/src/lib/components/explorer/TrashDialog.svelte index 992ce507..256d5f04 100644 --- a/web/src/lib/components/explorer/TrashDialog.svelte +++ b/web/src/lib/components/explorer/TrashDialog.svelte @@ -9,6 +9,7 @@ content: string; deletedAt: number; mergedInto?: string; + supersededBy?: string; projectName?: string; containerTag: string; }; @@ -72,6 +73,9 @@ {#if item.mergedInto} {i18n.t("trash-merged-into", { id: item.mergedInto })} {/if} + {#if item.supersededBy} + {i18n.t("trash-superseded-by", { id: item.supersededBy })} + {/if}