diff --git a/README.md b/README.md index 51c6369..21373f6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Optimize your coding-agent sessions with **DietCode** — automatic context comp Every time you submit a prompt, the plugin (via hooks in Claude Code and Codex CLI, or via rules guidance in Cursor): -1. **Classifies your intent** and prepends a one-line hint (e.g. `[DietCode intent: file_read (87%)]`) so the agent picks the right tool without guessing +1. **Classifies your intent** so the agent picks the right tool without guessing 2. **Compresses large contexts** automatically when you paste in a big codebase and ask a retrieval-style question — reducing token usage by 50–70% before the prompt reaches the model 3. **Compresses large tool outputs** (`PostToolUse`) — `ls`, `grep`, `git diff/log/status` are structurally compacted with zero latency, and anything still large is run through Scaledown before it enters context 4. **Tracks token savings** — every compression/summarization is counted, shown live in the Claude Code status line (`↓ 125.4K saved · 747 reqs`) diff --git a/src/client.ts b/src/client.ts index 940b49e..8e339f5 100644 --- a/src/client.ts +++ b/src/client.ts @@ -166,12 +166,17 @@ export class ScaledownClient { async summarize( text: string, instructions?: string, - maxTokens?: number + maxTokens?: number, + messages?: unknown[] ): Promise { return this.post("/summarization/abstractive", { text, ...(instructions !== undefined && { instructions }), ...(maxTokens !== undefined && { max_tokens: maxTokens }), + // Structured message blocks (tool_use/tool_result included), sent + // alongside the flattened `text` above so the backend can process tool + // output structurally rather than relying only on the flattened text. + ...(messages !== undefined && { messages }), }); } diff --git a/src/config.ts b/src/config.ts index ed28f5a..d9ff24c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -58,7 +58,7 @@ export function loadProxyConfig(): ProxyConfig { compactThreshold: intEnv("SCALEDOWN_PROXY_COMPACT_THRESHOLD", 50000), disable: process.env.SCALEDOWN_PROXY_DISABLE === "true", blockCompress: process.env.SCALEDOWN_PROXY_BLOCK_COMPRESS === "true", - foldEveryTurns: intEnv("SCALEDOWN_PROXY_FOLD_TURNS", 3), + foldEveryTurns: intEnv("SCALEDOWN_PROXY_FOLD_TURNS", 5), cacheControlDisable: process.env.SCALEDOWN_PROXY_CACHE_CONTROL_DISABLE === "true", }; } diff --git a/src/proxy/server.ts b/src/proxy/server.ts index 9ddbff6..0e76dbb 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -56,13 +56,20 @@ function outboundHeaders(req: IncomingMessage): Record { return headers; } +// Headers describing the upstream *wire* response that no longer apply once +// undici's fetch has already decompressed the body for us (upstream.body): +// forwarding content-encoding/content-length here would tell the client to +// decompress an already-plain body, which fails (e.g. BrotliDecompressionError +// in Claude Code's own fetch client). +const STRIP_RESPONSE_HEADERS = new Set(["content-encoding", "content-length"]); + async function pipeUpstream( upstream: Response, res: ServerResponse ): Promise { const headers: Record = {}; upstream.headers.forEach((value, key) => { - if (key.toLowerCase() === "content-length") return; // may change; let Node set + if (STRIP_RESPONSE_HEADERS.has(key.toLowerCase())) return; headers[key] = value; }); res.writeHead(upstream.status, headers); @@ -75,8 +82,12 @@ async function pipeUpstream( export function createProxyServer(config: Config, client: ScaledownClient): Server { const { proxy } = config; - const summarize = (text: string, instructions?: string) => - client.summarize(text, instructions).then((r) => r.summary); + const summarize = (text: string, instructions?: string, messages?: unknown[]) => + client.summarize(text, instructions, undefined, messages).then((r) => ({ + summary: r.summary, + inputChars: r.input_chars, + outputChars: r.output_chars, + })); return createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://localhost"); @@ -96,7 +107,8 @@ export function createProxyServer(config: Config, client: ScaledownClient): Serv if (result.compacted) { saveSessionState(sessionId, result.state); process.stderr.write( - `dietcode proxy: compaction step (session ${sessionId.slice(0, 8)}) — saved ~${result.savedTokens} tokens\n` + `dietcode proxy: compaction step (session ${sessionId.slice(0, 8)}) — ` + + `saved ~${result.savedTokens} tokens (~$${result.netSavingsUsd.toFixed(4)})\n` ); } if (result.savedTokens > 0) addSaving(sessionId, result.savedTokens); diff --git a/src/proxy/transform.test.ts b/src/proxy/transform.test.ts index c957fc7..905e5ef 100644 --- a/src/proxy/transform.test.ts +++ b/src/proxy/transform.test.ts @@ -33,6 +33,10 @@ const baseConfig: ProxyConfig = { const EMPTY: SessionState = { runningSummary: "", agedThrough: 0, updatedAt: "" }; +function summarizeResult(summary: string, inputChars = 0, outputChars = 0) { + return { summary, inputChars: inputChars || summary.length, outputChars: outputChars || summary.length }; +} + // 4 user prompts (idx 0,2,4,6) + assistants between. function sampleMessages(): MessagesBody["messages"] { return [ @@ -91,7 +95,7 @@ describe("under the compaction threshold", () => { describe("at a compaction step (over threshold)", () => { it("calls summarize exactly once, folds the summary in, and advances state", async () => { - const summarize = jest.fn().mockResolvedValue("FRESH SUMMARY"); + const summarize = jest.fn().mockResolvedValue(summarizeResult("FRESH SUMMARY")); const res = await transformRequest(body(), { ...EMPTY }, { ...baseConfig, compactThreshold: 1 }, { summarize, }); @@ -107,7 +111,7 @@ describe("at a compaction step (over threshold)", () => { }); it("asks ScaleDown to dedupe repeated tool output rather than describe it", async () => { - const summarize = jest.fn().mockResolvedValue("FRESH SUMMARY"); + const summarize = jest.fn().mockResolvedValue(summarizeResult("FRESH SUMMARY")); await transformRequest(body(), { ...EMPTY }, { ...baseConfig, compactThreshold: 1 }, { summarize }); const instructions = summarize.mock.calls[0][1] as string; @@ -115,7 +119,7 @@ describe("at a compaction step (over threshold)", () => { }); it("reports positive savings when the aged content is large", async () => { - const summarize = jest.fn().mockResolvedValue("TINY SUMMARY"); + const summarize = jest.fn().mockResolvedValue(summarizeResult("TINY SUMMARY")); const big = "lorem ipsum ".repeat(500); // ~6k chars per message const heavy: MessagesBody = { messages: [ @@ -136,7 +140,7 @@ describe("at a compaction step (over threshold)", () => { }); it("extends the prior summary rather than starting over", async () => { - const summarize = jest.fn().mockResolvedValue("MERGED"); + const summarize = jest.fn().mockResolvedValue(summarizeResult("MERGED")); const state: SessionState = { runningSummary: "PRIOR", agedThrough: 2, updatedAt: "" }; await transformRequest(body(), state, { ...baseConfig, compactThreshold: 1 }, { summarize }); const input = summarize.mock.calls[0][0] as string; @@ -144,7 +148,7 @@ describe("at a compaction step (over threshold)", () => { }); it("never touches system or tools", async () => { - const summarize = jest.fn().mockResolvedValue("S"); + const summarize = jest.fn().mockResolvedValue(summarizeResult("S")); const b = body(); const res = await transformRequest(b, { ...EMPTY }, { ...baseConfig, compactThreshold: 1 }, { summarize, @@ -198,7 +202,7 @@ describe("turn-cadence trigger", () => { }); it("folds once foldEveryTurns new turns have accumulated, even under the token threshold", async () => { - const summarize = jest.fn().mockResolvedValue("CADENCE SUMMARY"); + const summarize = jest.fn().mockResolvedValue(summarizeResult("CADENCE SUMMARY")); // 5 user prompts (idx 0,2,4,6,8); recentTurns=2 leaves 3 foldable turns. const many: MessagesBody = { messages: [ diff --git a/src/proxy/transform.ts b/src/proxy/transform.ts index b1bc06a..e4f06c0 100644 --- a/src/proxy/transform.ts +++ b/src/proxy/transform.ts @@ -1,21 +1,24 @@ // Progressive compaction for the DietCode proxy. // -// The key property: we do NOT summarize on every request. We keep a per-session -// "running summary" standing in for the oldest turns, plus a verbatim "hot -// window" of everything since. Most requests just forward (summary + hot -// window) with zero ScaleDown calls. A compaction step — one ScaleDown call -// that folds the older part of the hot window into the running summary — fires -// when EITHER (summary + hot window) exceeds `compactThreshold` (a safety net -// for a single turn that dumps a huge tool result) OR `foldEveryTurns` new -// turns have accumulated since the last fold (a steady cadence that keeps -// stale/duplicate tool output — repeated file reads, re-run commands — from -// sitting in the live window for long). `state.agedThrough` is the boundary: -// everything at or before it has already been folded into `runningSummary` by -// ScaleDown; only messages after it are live and eligible for the next fold. -// Deduplicating repeated tool output is instructions to ScaleDown's summarize -// call (see SUMMARY_INSTRUCTIONS below), not separate local logic — the fold -// input already contains every duplicate, and the model is told to collapse -// them itself. +// The key property: we do NOT call ScaleDown on every request. We keep a +// per-session "running summary" standing in for the oldest turns, plus a +// verbatim "hot window" of everything since. Most requests just forward +// (summary + hot window) with zero ScaleDown calls. A compaction step — one +// ScaleDown call that folds the older part of the hot window into the running +// summary — fires when EITHER (summary + hot window) exceeds +// `compactThreshold` (a safety net for a single turn that dumps a huge tool +// result) OR `foldEveryTurns` new turns have accumulated since the last fold +// (a steady cadence that keeps stale/duplicate tool output — repeated file +// reads, re-run commands — from sitting in the live window for long). +// `state.agedThrough` is the boundary: everything at or before it has already +// been folded into `runningSummary`; only messages after it are live and +// eligible for the next fold. +// +// The aged chunk is sent to ScaleDown as structured message blocks alongside +// the flattened text, so the backend can process tool_use/tool_result content +// structurally rather than relying only on flattened text. This file doesn't +// need to know how the backend uses that — only that the response shape +// (summary + input/output chars) is the same either way. // // Cache safety: `system`, `tools`, and the verbatim tail are never modified. // The folded summary block is byte-identical between compaction steps (it is @@ -42,19 +45,45 @@ export interface MessagesBody { [k: string]: unknown; } +export interface SummarizeResult { + summary: string; + /** Chars sent to ScaleDown for this fold — used to net the call's own cost out of savedTokens. */ + inputChars: number; + /** Chars ScaleDown returned — used to net the call's own cost out of savedTokens. */ + outputChars: number; +} + export interface TransformDeps { - /** Injected so tests can run without a live ScaledownClient. */ - summarize: (text: string, instructions?: string) => Promise; + /** Injected so tests can run without a live ScaledownClient. `messages` is + * the structured aged chunk (tool_use/tool_result blocks intact), sent + * alongside the flattened text so the backend can process it structurally. */ + summarize: ( + text: string, + instructions?: string, + messages?: AnthropicMessage[] + ) => Promise; } export interface TransformResult { body: MessagesBody; + /** Token-equivalent of netSavingsUsd (netSavingsUsd / ANTHROPIC_CACHE_READ_PER_TOKEN), for callers that still want a token figure. */ savedTokens: number; + /** Net dollar savings: Anthropic-side cost avoided by folding, minus the ScaleDown call's own dollar cost. Can be negative if a fold cost more than it saved. */ + netSavingsUsd: number; state: SessionState; /** True iff a ScaleDown call happened on this request (a compaction step). */ compacted: boolean; } +// Pricing for netting ScaleDown's call cost against the Anthropic tokens it +// saves — these are two very different rates (ScaleDown is ~0.05/MTok with no +// output charge; Anthropic cache read/write is priced per-token at a much +// higher rate), so netting raw TOKEN COUNTS 1:1 previously made an extremely +// cheap ScaleDown call look like it cost more than it saved even when the real +// dollar savings were strongly positive. Comparing in dollars fixes that. +const ANTHROPIC_CACHE_READ_PER_TOKEN = 0.3 / 1_000_000; +const SCALEDOWN_PER_TOKEN = 0.05 / 1_000_000; + const SUMMARY_INSTRUCTIONS = "Summarize this software-engineering conversation concisely, preserving key " + "decisions, code changes, exact file paths, commands, error messages, and any " + @@ -193,7 +222,7 @@ export async function transformRequest( ): Promise { const messages = body.messages; if (!Array.isArray(messages) || messages.length === 0) { - return { body, savedTokens: 0, state, compacted: false }; + return { body, savedTokens: 0, netSavingsUsd: 0, state, compacted: false }; } // Defensive: if history shrank below where we'd aged to (new/forked @@ -206,6 +235,13 @@ export async function transformRequest( let compacted = false; let retrieveId = state.runningSummary ? putOriginal(state.runningSummary, state.runningSummary) : ""; + // ScaleDown-side dollar cost of a compaction step's own summarize call — + // netted out of netSavingsUsd below so the reported number reflects total + // spend, not just the Anthropic-side message-size delta. Priced in dollars + // (not raw token counts) because ScaleDown and Anthropic tokens cost very + // different amounts per token — see ANTHROPIC_CACHE_READ_PER_TOKEN / + // SCALEDOWN_PER_TOKEN above. + let summarizeCallCostUsd = 0; // Decide whether this request needs a compaction step: either the live // window has crossed the token budget (safety net), or foldEveryTurns new @@ -226,12 +262,19 @@ export async function transformRequest( ? `[Existing summary]\n${working.runningSummary}\n\n[New turns]\n${serialize(newlyAged)}` : serialize(newlyAged); try { - const summary = await deps.summarize(input, SUMMARY_INSTRUCTIONS); + // Send structured blocks alongside the flattened `input` text — the + // backend can process tool_use/tool_result content structurally when + // `messages` is present, but still needs `input`/instructions for the + // existing-summary-merge framing either way. + const result = await deps.summarize(input, SUMMARY_INSTRUCTIONS, newlyAged); + const summary = result.summary; if (summary && summary.trim()) { // Store the full aged transcript for sd_retrieve reversibility. retrieveId = putOriginal(serialize(messages.slice(0, boundary)), summary); working = { runningSummary: summary, agedThrough: boundary, updatedAt: "" }; compacted = true; + const scaledownTokens = Math.ceil((result.inputChars + result.outputChars) / 4); + summarizeCallCostUsd = scaledownTokens * SCALEDOWN_PER_TOKEN; } } catch { // Fail-open: keep the prior state, forward without a new summary. @@ -246,14 +289,26 @@ export async function transformRequest( MAX_CACHE_BREAKPOINTS; const forwarded = applySummary(messages, working, retrieveId, addCacheControl); - const savedTokens = Math.max( - 0, - estimateTokens(JSON.stringify(messages)) - estimateTokens(JSON.stringify(forwarded)) - ); + // Anthropic-side tokens avoided by folding — priced as cache reads, since + // those tokens would otherwise sit in the live window and be re-read (at + // minimum) on every subsequent request. + const anthropicTokensAvoided = + estimateTokens(JSON.stringify(messages)) - estimateTokens(JSON.stringify(forwarded)); + const anthropicSavingsUsd = Math.max(0, anthropicTokensAvoided) * ANTHROPIC_CACHE_READ_PER_TOKEN; + // Net dollar savings = what folding avoided on the Anthropic side minus what + // the ScaleDown call itself cost. Comparing in dollars (not raw token + // counts) matters because the two are priced very differently — see + // ANTHROPIC_CACHE_READ_PER_TOKEN / SCALEDOWN_PER_TOKEN above. + const netSavingsUsd = anthropicSavingsUsd - summarizeCallCostUsd; + // Token-equivalent of netSavingsUsd, for callers that want a token figure + // rather than a dollar one. Clamped at 0: a fold that cost more (in real + // dollars) than it saved reports no savings rather than a negative token count. + const savedTokens = Math.max(0, Math.round(netSavingsUsd / ANTHROPIC_CACHE_READ_PER_TOKEN)); return { body: { ...body, messages: forwarded }, savedTokens, + netSavingsUsd, state: working, compacted, };