From d72bf6bd600e4029cc44369b19390820cb4dd57c Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:55:36 +0100 Subject: [PATCH 1/2] feat(capture): optional update-on-capture for similar memories Search for similar memories before auto-capture persist and let the model choose add, update, or supersede (soft-delete into trash). --- README.md | 34 +++- opencode-mem.example.jsonc | 13 +- src/config.ts | 72 +++++++- src/services/api-handlers.ts | 103 +---------- src/services/auto-capture.ts | 229 ++++++++++++++++++++++++- src/services/client.ts | 161 ++++++++++++++++- src/services/update-on-capture.ts | 186 ++++++++++++++++++++ tests/add-memory-phase-logging.test.ts | 1 + tests/config.test.ts | 8 + tests/update-on-capture.test.ts | 152 ++++++++++++++++ 10 files changed, 843 insertions(+), 116 deletions(-) create mode 100644 src/services/update-on-capture.ts create mode 100644 tests/update-on-capture.test.ts diff --git a/README.md b/README.md index 6515cab0..5d2500af 100644 --- a/README.md +++ b/README.md @@ -279,14 +279,36 @@ memories do not consume slots that could hold eligible memories. Setting `chatMessage.maxMemories` to `0` disables project-memory injection while still allowing the user profile to be injected when enabled. +### Update-on-capture (optional) + +Opt-in path that keeps memories current as facts change. When `updateOnCaptureEnabled` is +`true`, auto-capture embeds the new summary, searches for similar memories above +`updateOnCaptureSimilarityThreshold` (default `0.75`), and asks the capture model to +choose `add`, `update` (rewrite in place), or `supersede` (save new + soft-delete old into +trash). Pinned memories are skipped. Default is off so existing installs keep add-only +behavior. + +```jsonc +"updateOnCaptureEnabled": true, +"updateOnCaptureSimilarityThreshold": 0.75, +"updateOnCaptureMaxCandidates": 5, +``` + +Tip: with `Xenova/nomic-embed-text-v1`, enabling `embeddingUseTaskPrefixes: true` (and +optionally lowering recall `similarityThreshold`) often improves match quality for longer +memories. Prefixes stay opt-in by default. + ### Dreaming (optional memory consolidation) -Opt-in background pass that finds near-duplicate memories (same similarity threshold as -deduplication), asks the configured `opencodeProvider` / `opencodeModel` to merge them, -and soft-deletes superseded rows into recoverable trash. Pinned memories are never -touched. Configure under `dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). -The web UI exposes **Trash** (restore) and **Dream** (manual run). Soft-deleted rows are -hard-purged after `dreaming.trashRetentionDays` during normal cleanup. +Opt-in background pass that finds **related** memories (via `dreaming.relatedThreshold`, +default `0.75`, independent of `deduplicationSimilarityThreshold`), asks the configured +`opencodeProvider` / `opencodeModel` to `merge`, `update`, `supersede`, `keep`, or `flag` +contradictions, and soft-deletes superseded rows into recoverable trash. Pinned memories +are never touched. Set `dreaming.crossProject: true` (or `memory.defaultScope: +"all-projects"`) to group across project tags; user-scope stays separate. Configure under +`dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). The web UI exposes +**Trash** (restore) and **Dream** (manual run). Soft-deleted rows are hard-purged after +`dreaming.trashRetentionDays` during normal cleanup. ### Choosing / configuring embeddings diff --git a/opencode-mem.example.jsonc b/opencode-mem.example.jsonc index 801c7027..fd61a6c6 100644 --- a/opencode-mem.example.jsonc +++ b/opencode-mem.example.jsonc @@ -155,13 +155,16 @@ // "injectionMarkers": [""], }, - // Opt-in background consolidation ("dreaming"): merge near-duplicates via LLM - // and soft-delete superseded memories into recoverable trash. + // Opt-in background consolidation ("dreaming"): merge/update/supersede related + // memories via LLM and soft-delete superseded rows into recoverable trash. "dreaming": { "enabled": false, "intervalHours": 24, "maxGroupsPerRun": 5, "trashRetentionDays": 30, + "relatedThreshold": 0.75, + // Also effective when memory.defaultScope is "all-projects". + "crossProject": false, }, // ============================================ @@ -205,6 +208,12 @@ "autoCaptureEnabled": true, + // Opt-in: before saving a new auto-capture memory, search for similar entries + // and let the model choose add / update / supersede (old rows go to trash). + // "updateOnCaptureEnabled": true, + // "updateOnCaptureSimilarityThreshold": 0.75, + // "updateOnCaptureMaxCandidates": 5, + // When both opencodeProvider and manual memoryModel/memoryApiUrl are set, which // channel to try first for auto-capture summaries. Default "opencode-first". // Prefer "external-first" if the host session API is slow/busy and you want the diff --git a/src/config.ts b/src/config.ts index 2917e83b..1dd6e666 100644 --- a/src/config.ts +++ b/src/config.ts @@ -81,6 +81,16 @@ interface OpenCodeMemConfig { autoCaptureMaxRetries?: number; autoCaptureMaxContextBytes?: number; autoCaptureLanguage?: string; + /** + * When true, auto-capture searches for similar memories before insert and + * may update or supersede an existing entry instead of always adding. + * Default false (opt-in). + */ + updateOnCaptureEnabled?: boolean; + /** Cosine/hybrid floor for update-on-capture candidates. Default 0.75. */ + updateOnCaptureSimilarityThreshold?: number; + /** Max similar memories shown to the decision LLM. Default 5. */ + updateOnCaptureMaxCandidates?: number; /** * Which auto-capture summary channel to try first when both opencode and * manual (memoryModel/memoryApiUrl) providers are configured. @@ -239,6 +249,16 @@ interface OpenCodeMemConfig { intervalHours?: number; maxGroupsPerRun?: number; trashRetentionDays?: number; + /** + * Similarity floor for related (not only near-duplicate) dreaming groups. + * Default 0.75. Independent of deduplicationSimilarityThreshold. + */ + relatedThreshold?: number; + /** + * Allow grouping across project container tags. Also effective when + * memory.defaultScope is "all-projects". User-scope stays separate. + */ + crossProject?: boolean; }; } @@ -326,6 +346,9 @@ const DEFAULTS: Required< autoCaptureMaxRetries: 3, autoCaptureMaxContextBytes: 131072, autoCaptureProviderOrder: "opencode-first", + updateOnCaptureEnabled: false, + updateOnCaptureSimilarityThreshold: 0.75, + updateOnCaptureMaxCandidates: 5, aiSessionRetentionDays: 7, webServerEnabled: true, webServerPort: 4747, @@ -389,6 +412,8 @@ const DEFAULTS: Required< intervalHours: 24, maxGroupsPerRun: 5, trashRetentionDays: 30, + relatedThreshold: 0.75, + crossProject: false, }, }; @@ -600,6 +625,12 @@ const CONFIG_TEMPLATE = `{ // Note: Ollama may not support tool calling. Use OpenAI, Anthropic, or Groq for best results. "autoCaptureEnabled": true, + + // Opt-in: before saving a new auto-capture memory, search for similar entries + // and let the model choose add / update / supersede (old rows go to trash). + // "updateOnCaptureEnabled": true, + // "updateOnCaptureSimilarityThreshold": 0.75, + // "updateOnCaptureMaxCandidates": 5, // Provider type: "api-route" | "atlas-cloud" | "cheaper-inference" | "openai-chat" | "openai-responses" | "anthropic" | "minimax" | "orcarouter" | "runcrate" // Note: "openai-chat" is a generic OpenAI API-compatible mode. @@ -890,13 +921,18 @@ const CONFIG_TEMPLATE = `{ // "injectionMarkers": [""] }, - // Opt-in background consolidation: merge near-duplicate memories via the - // configured LLM and soft-delete superseded rows into trash (recoverable). + // Opt-in background consolidation: merge/update/supersede related memories + // via the configured LLM and soft-delete superseded rows into trash. "dreaming": { "enabled": false, "intervalHours": 24, "maxGroupsPerRun": 5, - "trashRetentionDays": 30 + "trashRetentionDays": 30, + // Related (not only near-duplicate) grouping floor; independent of + // deduplicationSimilarityThreshold (default 0.9). + "relatedThreshold": 0.75, + // Also effective when memory.defaultScope is "all-projects". + "crossProject": false } } `; @@ -1158,6 +1194,26 @@ export function normalizeDreamingTrashRetentionDays(value: unknown): number { return value; } +export function normalizeSimilarityUnitInterval(value: unknown, fallback: number): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + return fallback; + } + if (value < 0) return 0; + if (value > 1) return 1; + return value; +} + +export function normalizeUpdateOnCaptureMaxCandidates(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return 5; + } + return Math.max(1, Math.min(20, Math.trunc(value))); +} + +export function normalizeDreamingRelatedThreshold(value: unknown): number { + return normalizeSimilarityUnitInterval(value, 0.75); +} + /** * User-supplied markers extend the built-in set rather than replacing it, so * adding one marker cannot silently disable protection against all the others. @@ -1288,6 +1344,14 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { autoCaptureProviderOrder: normalizeAutoCaptureProviderOrder( fileConfig.autoCaptureProviderOrder ), + updateOnCaptureEnabled: fileConfig.updateOnCaptureEnabled ?? DEFAULTS.updateOnCaptureEnabled, + updateOnCaptureSimilarityThreshold: normalizeSimilarityUnitInterval( + fileConfig.updateOnCaptureSimilarityThreshold, + DEFAULTS.updateOnCaptureSimilarityThreshold + ), + updateOnCaptureMaxCandidates: normalizeUpdateOnCaptureMaxCandidates( + fileConfig.updateOnCaptureMaxCandidates + ), memoryProvider, memoryModel: fileConfig.memoryModel, memoryApiUrl: fileConfig.memoryApiUrl, @@ -1416,6 +1480,8 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { trashRetentionDays: normalizeDreamingTrashRetentionDays( fileConfig.dreaming?.trashRetentionDays ), + relatedThreshold: normalizeDreamingRelatedThreshold(fileConfig.dreaming?.relatedThreshold), + crossProject: fileConfig.dreaming?.crossProject ?? DEFAULTS.dreaming.crossProject, }, }; } diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 05f70a43..0ddc260a 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -3,7 +3,7 @@ import { tursoShardManager } from "./turso/shard-manager.js"; import { tursoVectorSearch } from "./turso/vector-search.js"; import { tursoConnectionManager } from "./turso/connection-manager.js"; import { ensureTursoReady } from "./turso/ready.js"; -import { formatTagsForEmbedding, parseSessionIdFromMetadata } from "./turso/vector-utils.js"; +import { formatTagsForEmbedding } from "./turso/vector-utils.js"; import { extractScopeFromContainerTag, tryExtractScopeFromContainerTag } from "./memory-scope.js"; import { log } from "./logger.js"; import { CONFIG } from "../config.js"; @@ -592,104 +592,11 @@ export async function handleUpdateMemory( data: { content?: string; type?: MemoryType; tags?: string[] } ): Promise> { try { - if (!id) return { success: false, error: "id is required" }; - await embeddingService.warmup(); - - // Find the existing memory first (read-only — no data modified yet) - const shards = await getAllMemoryShards(); - let foundShard = null, - existingMemory = null; - for (const shard of shards) { - const db = await tursoConnectionManager.getConnection(shard.dbPath); - const memory = await tursoVectorSearch.getMemoryById(db, id); - if (memory) { - foundShard = shard; - existingMemory = memory; - break; - } + const { memoryClient } = await import("./client.js"); + const result = await memoryClient.updateMemory(id, data); + if (!result.success) { + return { success: false, error: result.error || "Update failed" }; } - if (!foundShard || !existingMemory) return { success: false, error: "Memory not found" }; - - // STEP 1: Generate new embeddings FIRST (safe — no data deleted yet) - const newContent = data.content || String(existingMemory.content); - const existingTags = existingMemory.tags ? String(existingMemory.tags) : ""; - const tags = - data.tags || (existingTags ? existingTags.split(",").map((t: string) => t.trim()) : []); - const vector = await embeddingService.embedWithTimeout(newContent, { task: "document" }); - let tagsVector: Float32Array | undefined = undefined; - if (tags.length > 0) { - tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags), { - task: "document", - }); - } - - const db = await tursoConnectionManager.getConnection(foundShard.dbPath); - const updatedAt = Date.now(); - const memoryType = data.type || (existingMemory.type ? String(existingMemory.type) : undefined); - const containerTag = String(existingMemory.container_tag); - const existingMeta = safeJSONParse(existingMemory.metadata) || {}; - - if (isMarkdownBackend()) { - writeMemoryMarkdownFile({ - id, - content: newContent, - type: memoryType, - tags, - containerTag, - createdAt: Number(existingMemory.created_at), - updatedAt, - source: typeof existingMeta.source === "string" ? existingMeta.source : "api", - title: existingMemory.display_name ? String(existingMemory.display_name) : undefined, - }); - existingMeta.markdownRelPath = pluginMemoryRelPath(id); - } - - const metadataJson = JSON.stringify(existingMeta); - - const isPinned = Number(existingMemory.is_pinned ?? 0) === 1 ? 1 : 0; - const deletedAt = - existingMemory.deleted_at == null || existingMemory.deleted_at === undefined - ? null - : Number(existingMemory.deleted_at); - - await db.transaction("write", async (tx) => { - await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [id] }); - await tx.execute({ - sql: ` - INSERT INTO memories ( - id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, - metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url, - is_pinned, deleted_at - ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `, - args: [ - id, - newContent, - JSON.stringify(Array.from(vector)), - ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), - containerTag, - tags.length > 0 ? tags.join(",") : null, - memoryType || null, - Number(existingMemory.created_at), - updatedAt, - metadataJson, - existingMemory.session_id - ? String(existingMemory.session_id) - : parseSessionIdFromMetadata( - existingMemory.metadata ? String(existingMemory.metadata) : null - ), - existingMemory.display_name ? String(existingMemory.display_name) : null, - existingMemory.user_name ? String(existingMemory.user_name) : null, - existingMemory.user_email ? String(existingMemory.user_email) : null, - existingMemory.project_path ? String(existingMemory.project_path) : null, - existingMemory.project_name ? String(existingMemory.project_name) : null, - existingMemory.git_repo_url ? String(existingMemory.git_repo_url) : null, - isPinned, - deletedAt, - ], - }); - }); - return { success: true }; } catch (error) { log("handleUpdateMemory: error", { error: String(error) }); diff --git a/src/services/auto-capture.ts b/src/services/auto-capture.ts index 2987e1cd..559ea4b6 100644 --- a/src/services/auto-capture.ts +++ b/src/services/auto-capture.ts @@ -7,6 +7,13 @@ import { CONFIG } from "../config.js"; import { userPromptManager, type UserPrompt } from "./user-prompt/user-prompt-manager.js"; import { loadOpencodeProvider } from "./ai/opencode-provider-loader.js"; import { truncateToMaxBytes, utf8ByteLength } from "../utils/context-limit.js"; +import { + applyUpdateOnCaptureDecision, + buildUpdateOnCaptureDecisionPrompt, + filterUpdateOnCaptureCandidates, + type UpdateOnCaptureCandidate, + type UpdateOnCaptureDecision, +} from "./update-on-capture.js"; interface ToolCallInfo { name: string; @@ -168,8 +175,8 @@ async function capturePrompt( : summaryResult.summary; phase = "persist"; - const result = await memoryClient.addMemory(summaryWithTags, tags.project.tag, { - source: "auto-capture" as any, + const persistMeta = { + source: "auto-capture" as const, type: summaryResult.type as any, tags: summaryResult.tags, sessionID, @@ -181,16 +188,28 @@ async function capturePrompt( projectPath: tags.project.projectPath, projectName: tags.project.projectName, gitRepoUrl: tags.project.gitRepoUrl, - }); + }; + + const result = await persistAutoCaptureMemory( + ctx, + sessionID, + prompt, + summaryWithTags, + summaryResult.type, + summaryResult.tags || [], + tags.project.tag, + persistMeta + ); if (result.success) { - await userPromptManager.linkMemoryToPrompt(prompt.id, result.id); + await userPromptManager.linkMemoryToPrompt(prompt.id, result.id!); await userPromptManager.markAsCaptured(prompt.id); claimedPromptId = null; log("Auto-capture memory persisted", { promptId: prompt.id, sessionID, memoryId: result.id, + action: result.action, }); if (CONFIG.showAutoCaptureToasts) { @@ -712,6 +731,208 @@ async function tryExternalSummary( }; } +async function persistAutoCaptureMemory( + ctx: PluginInput, + sessionID: string, + prompt: UserPrompt, + summaryWithTags: string, + type: string, + tags: string[], + containerTag: string, + metadata: Record +): Promise<{ success: boolean; id?: string; error?: string; action: string }> { + if (!CONFIG.updateOnCaptureEnabled) { + const result = await memoryClient.addMemory(summaryWithTags, containerTag, metadata as any); + return { + success: result.success, + id: result.success ? result.id : undefined, + error: result.success ? undefined : result.error, + action: "add", + }; + } + + const scope = CONFIG.memory.defaultScope === "all-projects" ? "all-projects" : "project"; + const search = await memoryClient.searchMemories(summaryWithTags, containerTag, scope, { + similarityThreshold: CONFIG.updateOnCaptureSimilarityThreshold, + limit: CONFIG.updateOnCaptureMaxCandidates, + }); + + const rawCandidates: UpdateOnCaptureCandidate[] = (search.success ? search.results : []).map( + (r: any) => ({ + id: String(r.id), + content: String(r.memory ?? r.content ?? ""), + similarity: Number(r.similarity ?? 0), + isPinned: Number(r.isPinned ?? 0) === 1 || r.isPinned === true, + }) + ); + const candidates = filterUpdateOnCaptureCandidates(rawCandidates); + + if (candidates.length === 0) { + const result = await memoryClient.addMemory(summaryWithTags, containerTag, metadata as any); + return { + success: result.success, + id: result.success ? result.id : undefined, + error: result.success ? undefined : result.error, + action: "add", + }; + } + + let decision: UpdateOnCaptureDecision; + try { + decision = await decideUpdateOnCapture(ctx, sessionID, prompt, summaryWithTags, candidates); + } catch (error) { + log("update-on-capture: decision failed, falling back to add", serializeError(error)); + decision = { action: "add" }; + } + + return applyUpdateOnCaptureDecision( + memoryClient as any, + decision, + { + summary: summaryWithTags, + type, + tags, + containerTag, + metadata: metadata as any, + }, + new Set(candidates.map((c) => c.id)) + ); +} + +async function decideUpdateOnCapture( + ctx: PluginInput, + sessionID: string, + prompt: UserPrompt, + summary: string, + candidates: UpdateOnCaptureCandidate[] +): Promise { + const order = CONFIG.autoCaptureProviderOrder ?? "opencode-first"; + const channels: CaptureChannel[] = + order === "external-first" ? ["external", "opencode"] : ["opencode", "external"]; + + let lastError: unknown; + for (let i = 0; i < channels.length; i++) { + const channel = channels[i]!; + if (!isChannelConfigured(channel)) continue; + try { + return channel === "opencode" + ? await tryOpencodeUpdateDecision(summary, candidates, prompt) + : await tryExternalUpdateDecision(summary, candidates, sessionID, prompt); + } catch (error) { + lastError = error; + const nextConfigured = channels.slice(i + 1).find((c) => isChannelConfigured(c)); + if (!nextConfigured) break; + showFallbackToast(ctx, channel, error); + } + } + if (lastError) throw lastError; + return { action: "add" }; +} + +async function tryOpencodeUpdateDecision( + summary: string, + candidates: UpdateOnCaptureCandidate[], + prompt?: { id?: string; providerId: string | null; modelId: string | null } +): Promise { + const { ensureProviderConnected, getV2Client, generateStructuredOutput } = + await loadOpencodeProvider(); + + let providerID = CONFIG.opencodeProvider!; + let modelID = CONFIG.opencodeModel!; + if (modelID === "inherit") { + if (!prompt?.providerId || !prompt?.modelId) { + throw new Error( + "opencode-mem: opencodeModel is 'inherit' but no session model was recorded for this prompt" + ); + } + providerID = prompt.providerId; + modelID = prompt.modelId; + } + + if (!(await ensureProviderConnected(providerID))) { + throw new Error(`opencode provider '${providerID}' is not connected`); + } + const v2Client = getV2Client(); + if (!v2Client) { + throw new Error("opencode-mem: v2 client not initialized"); + } + + const { z } = await import("zod"); + const schema = z.object({ + action: z.enum(["add", "update", "supersede"]), + targetId: z.string().optional(), + content: z.string().optional(), + reason: z.string().optional(), + }); + + return generateStructuredOutput({ + client: v2Client, + providerID, + modelID, + ...(CONFIG.opencodeVariant ? { variant: CONFIG.opencodeVariant } : {}), + systemPrompt: + "You decide whether a new memory should add, update, or supersede similar existing memories. Return only structured JSON.", + userPrompt: buildUpdateOnCaptureDecisionPrompt(summary, candidates), + schema, + }); +} + +async function tryExternalUpdateDecision( + summary: string, + candidates: UpdateOnCaptureCandidate[], + sessionID: string, + prompt?: { id?: string; providerId: string | null; modelId: string | null } +): Promise { + const { AIProviderFactory } = await import("./ai/ai-provider-factory.js"); + const { buildMemoryProviderConfig } = await import("./ai/provider-config.js"); + + const providerConfig = buildMemoryProviderConfig(CONFIG); + const provider = AIProviderFactory.createProvider(CONFIG.memoryProvider, providerConfig); + const toolSchema = { + type: "function" as const, + function: { + name: "decide_memory_write", + description: "Decide how to store a new memory relative to similar existing ones", + parameters: { + type: "object", + properties: { + action: { + type: "string", + enum: ["add", "update", "supersede"], + }, + targetId: { type: "string" }, + content: { type: "string" }, + reason: { type: "string" }, + }, + required: ["action"], + }, + }, + }; + + const captureSessionID = `update-on-capture-${prompt?.id ?? sessionID}-${randomUUID()}`; + const result = await provider.executeToolCall( + "You decide whether a new memory should add, update, or supersede similar existing memories.", + buildUpdateOnCaptureDecisionPrompt(summary, candidates), + toolSchema, + captureSessionID + ); + + if (!result.success || !result.data) { + throw new Error(result.error || "Failed to decide update-on-capture action"); + } + + const action = result.data.action; + if (action !== "add" && action !== "update" && action !== "supersede") { + return { action: "add" }; + } + return { + action, + targetId: typeof result.data.targetId === "string" ? result.data.targetId : undefined, + content: typeof result.data.content === "string" ? result.data.content : undefined, + reason: typeof result.data.reason === "string" ? result.data.reason : undefined, + }; +} + async function generateSummary( ctx: PluginInput, context: string, diff --git a/src/services/client.ts b/src/services/client.ts index 12a00d1a..5409396b 100644 --- a/src/services/client.ts +++ b/src/services/client.ts @@ -3,7 +3,7 @@ import { tursoShardManager } from "./turso/shard-manager.js"; import { tursoVectorSearch } from "./turso/vector-search.js"; import { tursoConnectionManager } from "./turso/connection-manager.js"; import { ensureTursoReady } from "./turso/ready.js"; -import { formatTagsForEmbedding } from "./turso/vector-utils.js"; +import { formatTagsForEmbedding, parseSessionIdFromMetadata } from "./turso/vector-utils.js"; import { extractScopeFromContainerTag, resolveMemoryScope, @@ -279,7 +279,128 @@ export class LocalMemoryClient { } } - async deleteMemory(memoryId: string) { + async updateMemory( + id: string, + data: { + content?: string; + type?: MemoryType; + tags?: string[]; + metadataPatch?: Record; + } + ) { + try { + if (!id) return { success: false as const, error: "id is required" }; + await this.initialize(); + await embeddingService.warmup(); + + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); + const shards = [...userShards, ...projectShards]; + + let foundShard = null; + let existingMemory = null; + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id); + if (memory) { + foundShard = shard; + existingMemory = memory; + break; + } + } + if (!foundShard || !existingMemory) { + return { success: false as const, error: "Memory not found" }; + } + + const newContent = data.content || String(existingMemory.content); + const existingTags = existingMemory.tags ? String(existingMemory.tags) : ""; + const tags = + data.tags || (existingTags ? existingTags.split(",").map((t: string) => t.trim()) : []); + const vector = await embeddingService.embedWithTimeout(newContent, { task: "document" }); + let tagsVector: Float32Array | undefined = undefined; + if (tags.length > 0) { + tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags), { + task: "document", + }); + } + + const db = await tursoConnectionManager.getConnection(foundShard.dbPath); + const updatedAt = Date.now(); + const memoryType = + data.type || (existingMemory.type ? String(existingMemory.type) : undefined); + const containerTag = String(existingMemory.container_tag); + const existingMeta = safeJSONParse(existingMemory.metadata) || {}; + if (data.metadataPatch) Object.assign(existingMeta, data.metadataPatch); + + if (isMarkdownBackend()) { + writeMemoryMarkdownFile({ + id, + content: newContent, + type: memoryType, + tags, + containerTag, + createdAt: Number(existingMemory.created_at), + updatedAt, + source: typeof existingMeta.source === "string" ? existingMeta.source : "api", + title: existingMemory.display_name ? String(existingMemory.display_name) : undefined, + }); + existingMeta.markdownRelPath = pluginMemoryRelPath(id); + } + + const metadataJson = JSON.stringify(existingMeta); + const isPinned = Number(existingMemory.is_pinned ?? 0) === 1 ? 1 : 0; + const deletedAt = + existingMemory.deleted_at == null || existingMemory.deleted_at === undefined + ? null + : Number(existingMemory.deleted_at); + + await db.transaction("write", async (tx) => { + await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [id] }); + await tx.execute({ + sql: ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, session_id, display_name, user_name, user_email, project_path, project_name, git_repo_url, + is_pinned, deleted_at + ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + args: [ + id, + newContent, + JSON.stringify(Array.from(vector)), + ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), + containerTag, + tags.length > 0 ? tags.join(",") : null, + memoryType || null, + Number(existingMemory.created_at), + updatedAt, + metadataJson, + existingMemory.session_id + ? String(existingMemory.session_id) + : parseSessionIdFromMetadata( + existingMemory.metadata ? String(existingMemory.metadata) : null + ), + existingMemory.display_name ? String(existingMemory.display_name) : null, + existingMemory.user_name ? String(existingMemory.user_name) : null, + existingMemory.user_email ? String(existingMemory.user_email) : null, + existingMemory.project_path ? String(existingMemory.project_path) : null, + existingMemory.project_name ? String(existingMemory.project_name) : null, + existingMemory.git_repo_url ? String(existingMemory.git_repo_url) : null, + isPinned, + deletedAt, + ], + }); + }); + + return { success: true as const }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + log("updateMemory: error", { id, error: errorMessage }); + return { success: false as const, error: errorMessage }; + } + } + + async deleteMemory(memoryId: string, metaPatch?: Record) { try { await this.initialize(); @@ -293,7 +414,7 @@ export class LocalMemoryClient { if (memory) { // Soft-delete into trash (recoverable). Hard-purge happens via cleanup. - const ok = await tursoVectorSearch.softDeleteVector(db, memoryId); + const ok = await tursoVectorSearch.softDeleteVector(db, memoryId, metaPatch); return ok ? { success: true } : { success: false, error: "Memory not found" }; } } @@ -306,6 +427,40 @@ export class LocalMemoryClient { } } + async getMemoryById(memoryId: string) { + try { + await this.initialize(); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); + for (const shard of [...userShards, ...projectShards]) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, memoryId); + if (memory) { + return { + success: true as const, + memory: { + id: String(memory.id), + content: String(memory.content), + isPinned: Number(memory.is_pinned ?? 0) === 1, + containerTag: String(memory.container_tag), + type: memory.type ? String(memory.type) : undefined, + tags: memory.tags + ? String(memory.tags) + .split(",") + .map((t) => t.trim()) + .filter(Boolean) + : [], + }, + }; + } + } + return { success: false as const, error: "Memory not found" }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { success: false as const, error: errorMessage }; + } + } + async listMemories( containerTag: string, limit = 20, diff --git a/src/services/update-on-capture.ts b/src/services/update-on-capture.ts new file mode 100644 index 00000000..ee8d0489 --- /dev/null +++ b/src/services/update-on-capture.ts @@ -0,0 +1,186 @@ +import { CONFIG } from "../config.js"; +import { log } from "./logger.js"; +import type { MemoryType } from "../types/index.js"; + +export type UpdateOnCaptureAction = "add" | "update" | "supersede"; + +export interface UpdateOnCaptureCandidate { + id: string; + content: string; + similarity: number; + isPinned?: boolean; +} + +export interface UpdateOnCaptureDecision { + action: UpdateOnCaptureAction; + targetId?: string; + content?: string; + reason?: string; +} + +export type UpdateOnCaptureMemoryClient = { + addMemory: ( + content: string, + containerTag: string, + metadata?: Record + ) => Promise<{ success: boolean; id?: string; error?: string }>; + updateMemory: ( + id: string, + data: { + content?: string; + type?: MemoryType; + tags?: string[]; + metadataPatch?: Record; + } + ) => Promise<{ success: boolean; error?: string }>; + deleteMemory: ( + memoryId: string, + metaPatch?: Record + ) => Promise<{ success: boolean; error?: string }>; + getMemoryById: (memoryId: string) => Promise<{ + success: boolean; + memory?: { id: string; content: string; isPinned: boolean }; + error?: string; + }>; +}; + +export interface PersistCaptureInput { + summary: string; + type: string; + tags: string[]; + containerTag: string; + metadata: { + source: "auto-capture"; + type: MemoryType | string; + tags: string[]; + sessionID: string; + promptId: string; + captureTimestamp: number; + displayName?: string; + userName?: string; + userEmail?: string; + projectPath?: string; + projectName?: string; + gitRepoUrl?: string; + }; +} + +/** Drop pinned rows and truncate to the configured candidate cap. */ +export function filterUpdateOnCaptureCandidates( + candidates: UpdateOnCaptureCandidate[], + maxCandidates = CONFIG.updateOnCaptureMaxCandidates +): UpdateOnCaptureCandidate[] { + return candidates + .filter((c) => !c.isPinned) + .sort((a, b) => b.similarity - a.similarity) + .slice(0, Math.max(1, maxCandidates)); +} + +export function buildUpdateOnCaptureDecisionPrompt( + summary: string, + candidates: UpdateOnCaptureCandidate[] +): string { + const items = candidates.map((c) => ({ + id: c.id, + similarity: Number(c.similarity.toFixed(3)), + content: c.content.length > 800 ? `${c.content.slice(0, 800)}…` : c.content, + })); + return `You decide how to store a new auto-captured memory relative to similar existing memories. + +New memory draft: +${summary} + +Similar existing memories (JSON): +${JSON.stringify(items, null, 2)} + +Choose exactly one action: +- "add": new distinct fact; keep existing memories unchanged. +- "update": rewrite targetId in place with the corrected/current content (use "content"). +- "supersede": save the new memory and soft-delete targetId (outdated fact replaced). + +Rules: +- Prefer "add" when unsure or when memories are only loosely related. +- Use "update" / "supersede" only when the new draft clearly updates or replaces the same fact. +- targetId MUST be one of the candidate ids when action is update or supersede. +- For "update", "content" MUST retain important details (paths, commands, why something failed). +- Do NOT invent facts. Do NOT touch pinned memories (already excluded). +- Return structured JSON only.`; +} + +/** + * Apply an LLM decision. Invalid update/supersede targets fall back to add. + * Returns the persisted memory id (new or updated). + */ +export async function applyUpdateOnCaptureDecision( + client: UpdateOnCaptureMemoryClient, + decision: UpdateOnCaptureDecision, + input: PersistCaptureInput, + candidateIds: Set +): Promise<{ success: boolean; id?: string; action: UpdateOnCaptureAction; error?: string }> { + const fallbackAdd = async () => { + const result = await client.addMemory(input.summary, input.containerTag, input.metadata as any); + if (!result.success) { + return { success: false as const, action: "add" as const, error: result.error }; + } + return { success: true as const, id: result.id, action: "add" as const }; + }; + + const action = decision.action || "add"; + if (action === "add") { + return fallbackAdd(); + } + + const targetId = decision.targetId; + if (!targetId || !candidateIds.has(targetId)) { + log("update-on-capture: invalid targetId, falling back to add", { + action, + targetId, + }); + return fallbackAdd(); + } + + const existing = await client.getMemoryById(targetId); + if (!existing.success || !existing.memory || existing.memory.isPinned) { + log("update-on-capture: target missing or pinned, falling back to add", { + action, + targetId, + }); + return fallbackAdd(); + } + + if (action === "update") { + const content = (decision.content || input.summary).trim(); + if (!content) return fallbackAdd(); + const updated = await client.updateMemory(targetId, { + content, + type: input.type as MemoryType, + tags: input.tags, + metadataPatch: { + source: "auto-capture", + updatedByCapture: true, + promptId: input.metadata.promptId, + sessionID: input.metadata.sessionID, + captureTimestamp: input.metadata.captureTimestamp, + }, + }); + if (!updated.success) { + return { success: false, action: "update", error: updated.error }; + } + return { success: true, id: targetId, action: "update" }; + } + + // supersede + const added = await client.addMemory(input.summary, input.containerTag, input.metadata as any); + if (!added.success) { + return { success: false, action: "supersede", error: added.error }; + } + const deleted = await client.deleteMemory(targetId, { supersededBy: added.id }); + if (!deleted.success) { + log("update-on-capture: supersede soft-delete failed", { + targetId, + newId: added.id, + error: deleted.error, + }); + } + return { success: true, id: added.id, action: "supersede" }; +} diff --git a/tests/add-memory-phase-logging.test.ts b/tests/add-memory-phase-logging.test.ts index 52d84695..f600c05c 100644 --- a/tests/add-memory-phase-logging.test.ts +++ b/tests/add-memory-phase-logging.test.ts @@ -60,6 +60,7 @@ mock.module(${JSON.stringify(embeddingUrl)}, () => ({ mock.module(${JSON.stringify(vectorUtilsUrl)}, () => ({ formatTagsForEmbedding: (tags) => tags.join(" "), + parseSessionIdFromMetadata: () => undefined, })); mock.module(${JSON.stringify(shardManagerUrl)}, () => ({ diff --git a/tests/config.test.ts b/tests/config.test.ts index 9e7beceb..4ea67fda 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -118,6 +118,14 @@ describe("config", () => { expect(CONFIG.dreaming.intervalHours).toBe(24); expect(CONFIG.dreaming.maxGroupsPerRun).toBe(5); expect(CONFIG.dreaming.trashRetentionDays).toBe(30); + expect(CONFIG.dreaming.relatedThreshold).toBe(0.75); + expect(CONFIG.dreaming.crossProject).toBe(false); + }); + + it("should default update-on-capture to disabled", () => { + expect(CONFIG.updateOnCaptureEnabled).toBe(false); + expect(CONFIG.updateOnCaptureSimilarityThreshold).toBe(0.75); + expect(CONFIG.updateOnCaptureMaxCandidates).toBe(5); }); it("should have boolean toggle settings", () => { diff --git a/tests/update-on-capture.test.ts b/tests/update-on-capture.test.ts new file mode 100644 index 00000000..8f5f9c3f --- /dev/null +++ b/tests/update-on-capture.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "bun:test"; +import { + normalizeSimilarityUnitInterval, + normalizeUpdateOnCaptureMaxCandidates, +} from "../src/config.js"; +import { + applyUpdateOnCaptureDecision, + filterUpdateOnCaptureCandidates, + type UpdateOnCaptureMemoryClient, +} from "../src/services/update-on-capture.js"; + +describe("update-on-capture config helpers", () => { + it("clamps similarity thresholds to [0,1]", () => { + expect(normalizeSimilarityUnitInterval(0.75, 0.5)).toBe(0.75); + expect(normalizeSimilarityUnitInterval(-1, 0.5)).toBe(0); + expect(normalizeSimilarityUnitInterval(2, 0.5)).toBe(1); + expect(normalizeSimilarityUnitInterval("x", 0.5)).toBe(0.5); + }); + + it("normalizes max candidates", () => { + expect(normalizeUpdateOnCaptureMaxCandidates(3)).toBe(3); + expect(normalizeUpdateOnCaptureMaxCandidates(0)).toBe(5); + expect(normalizeUpdateOnCaptureMaxCandidates(99)).toBe(20); + }); +}); + +describe("filterUpdateOnCaptureCandidates", () => { + it("drops pinned and respects max", () => { + const out = filterUpdateOnCaptureCandidates( + [ + { id: "a", content: "a", similarity: 0.9, isPinned: true }, + { id: "b", content: "b", similarity: 0.8 }, + { id: "c", content: "c", similarity: 0.85 }, + { id: "d", content: "d", similarity: 0.7 }, + ], + 2 + ); + expect(out.map((c) => c.id)).toEqual(["c", "b"]); + }); +}); + +describe("applyUpdateOnCaptureDecision", () => { + const baseInput = { + summary: "KyuShared is on sda5", + type: "configuration", + tags: ["linux"], + containerTag: "project_abc", + metadata: { + source: "auto-capture" as const, + type: "configuration", + tags: ["linux"], + sessionID: "s1", + promptId: "p1", + captureTimestamp: 1, + }, + }; + + function mockClient( + overrides: Partial = {} + ): UpdateOnCaptureMemoryClient & { + calls: string[]; + } { + const calls: string[] = []; + return { + calls, + addMemory: async () => { + calls.push("add"); + return { success: true, id: "mem_new" }; + }, + updateMemory: async (id) => { + calls.push(`update:${id}`); + return { success: true }; + }, + deleteMemory: async (id, patch) => { + calls.push(`delete:${id}:${JSON.stringify(patch)}`); + return { success: true }; + }, + getMemoryById: async (id) => { + calls.push(`get:${id}`); + return { success: true, memory: { id, content: "old", isPinned: false } }; + }, + ...overrides, + }; + } + + it("adds when action is add", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "add" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result).toEqual({ success: true, id: "mem_new", action: "add" }); + expect(client.calls).toEqual(["add"]); + }); + + it("updates a valid candidate", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "update", targetId: "mem_old", content: "rewritten" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.action).toBe("update"); + expect(result.id).toBe("mem_old"); + expect(client.calls[0]).toBe("get:mem_old"); + expect(client.calls[1]).toBe("update:mem_old"); + }); + + it("supersedes by adding then soft-deleting", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "supersede", targetId: "mem_old" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result).toEqual({ success: true, id: "mem_new", action: "supersede" }); + expect(client.calls).toContain("add"); + expect(client.calls.some((c) => c.startsWith("delete:mem_old:"))).toBe(true); + }); + + it("falls back to add for invalid target", async () => { + const client = mockClient(); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "update", targetId: "missing" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.action).toBe("add"); + expect(client.calls).toEqual(["add"]); + }); + + it("falls back to add when target is pinned", async () => { + const client = mockClient({ + getMemoryById: async (id) => ({ + success: true, + memory: { id, content: "old", isPinned: true }, + }), + }); + const result = await applyUpdateOnCaptureDecision( + client, + { action: "update", targetId: "mem_old", content: "x" }, + baseInput, + new Set(["mem_old"]) + ); + expect(result.action).toBe("add"); + }); +}); From 913740ef71c040aba3d9891a8f1df21b2ae95169 Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:57:56 +0100 Subject: [PATCH 2/2] docs: keep PR1 dreaming README accurate until deeper dreaming lands --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5d2500af..e4b92b34 100644 --- a/README.md +++ b/README.md @@ -300,15 +300,15 @@ memories. Prefixes stay opt-in by default. ### Dreaming (optional memory consolidation) -Opt-in background pass that finds **related** memories (via `dreaming.relatedThreshold`, -default `0.75`, independent of `deduplicationSimilarityThreshold`), asks the configured -`opencodeProvider` / `opencodeModel` to `merge`, `update`, `supersede`, `keep`, or `flag` -contradictions, and soft-deletes superseded rows into recoverable trash. Pinned memories -are never touched. Set `dreaming.crossProject: true` (or `memory.defaultScope: -"all-projects"`) to group across project tags; user-scope stays separate. Configure under -`dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). The web UI exposes -**Trash** (restore) and **Dream** (manual run). Soft-deleted rows are hard-purged after -`dreaming.trashRetentionDays` during normal cleanup. +Opt-in background pass that finds near-duplicate memories (same similarity threshold as +deduplication), asks the configured `opencodeProvider` / `opencodeModel` to merge them, +and soft-deletes superseded rows into recoverable trash. Pinned memories are never +touched. Configure under `dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). +The web UI exposes **Trash** (restore) and **Dream** (manual run). Soft-deleted rows are +hard-purged after `dreaming.trashRetentionDays` during normal cleanup. + +Deeper related-threshold dreaming (update/supersede/flag, cross-project) lands in the +follow-up PR for #464. ### Choosing / configuring embeddings