diff --git a/CHANGELOG.md b/CHANGELOG.md index ddd8faba7..917c2a36b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ the archive; if it broke production, it belongs in `docs/incidents/`. See ## [Unreleased] +- **QA** — three regressions found in code shipped the day before, and a guard taught to read prose — backend, mobile, shared · [details](docs/changelog-archive/2026-H2.md#2026-09-12-qa-three-regressions-in-code-shipped-the-day-before) - **Guests** — a sign-in that left your reading behind no longer tells you it was kept — web, mobile, shared · [details](docs/changelog-archive/2026-H2.md#2026-09-11-guests-the-reader-is-told-when-their-work-did-not-come-across) - **Sentry** — a laptop's stale API key had been filing production-looking incidents for a month — backend, mobile · [details](docs/changelog-archive/2026-H2.md#2026-09-11-sentry-a-developer-machine-is-not-an-incident) - **Genres** — opening any genre on the phone showed nothing, because the endpoint never sent the authors both clients read — backend, mobile, shared · [details](docs/changelog-archive/2026-H2.md#2026-09-11-genres-a-field-two-clients-used-and-the-server-never-sent) diff --git a/apps/mobile/app/(auth)/login.tsx b/apps/mobile/app/(auth)/login.tsx index 676f4aeec..6b2ca0060 100644 --- a/apps/mobile/app/(auth)/login.tsx +++ b/apps/mobile/app/(auth)/login.tsx @@ -247,6 +247,7 @@ export default function LoginScreen() { ) await signInWithTokens(result.accessToken, result.refreshToken, result.user) + warnIfNothingCarried(result.guestMergeSkipped) if (isFreshAccount(result.user.createdAt)) trackSignUp('apple') else trackLogin('apple') landAfterAuth(result.user) diff --git a/apps/mobile/src/lib/guestMergeWarningLiterals.test.ts b/apps/mobile/src/lib/guestMergeWarningLiterals.test.ts new file mode 100644 index 000000000..360fa638b --- /dev/null +++ b/apps/mobile/src/lib/guestMergeWarningLiterals.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +/** + * Every sign-in that can drop a guest session must say so. + * + *

The server has reported guestMergeSkipped since guest sessions shipped and no client + * read it (ADR-014). #609 finally read it — on web, in authToastFor, and on mobile in + * app/(auth)/login.tsx's warnIfNothingCarried. The mobile side is a hand-written call + * at each entry point, which means it can be forgotten at one of them, and was.

+ * + *

There are exactly FOUR merge entry points — /auth/register, /auth/login, + * /auth/google, /auth/apple — because those are the four the server consults + * ResolveGuestToken on. All four go through mobilePost, which attaches the guest + * bearer, so all four can produce a skip. A screen test would catch this properly; mobile has none + * and vitest.config.ts is scoped to src/lib/** on purpose, so this greps instead — + * same shape as routeLiterals.test.ts and capabilityLiterals.test.ts.

+ * + *

KNOWN_GAPS is the finding. An entry point listed there is one that currently signs the + * reader in without telling them their earlier work stayed behind. Shrinking the list is always the + * fix; growing it needs a reason on the line. It was written with one entry — Apple — found by an + * adversarial QA pass the day after #609 shipped, and emptied the same day.

+ */ + +const LOGIN_SCREEN = join(__dirname, '..', '..', 'app', '(auth)', 'login.tsx') + +/** How far after the API call the warning may appear. The handlers are short; 15 is generous. */ +const WINDOW = 15 + +const ENTRY_POINTS = [ + 'registerWithEmail', + 'loginWithEmail', + 'loginWithGoogle', + 'loginWithApple', +] as const + +/** + * Entry points that do NOT warn today. Each is a bug, recorded rather than hidden. + * + *

Empty, and it should stay that way. `loginWithApple` sat here for a few hours on 2026-09-12: + * `handleAppleSignIn` called `signInWithTokens` + `landAfterAuth` and never `warnIfNothingCarried`, + * so an iOS reader whose guest row was not merged was landed in a new account with no indication + * that their highlights, vocabulary and progress had stayed behind. The other three paths in the + * same file warned. It was written three-quarters right and shipped that way.

+ */ +const KNOWN_GAPS: { entry: (typeof ENTRY_POINTS)[number]; why: string }[] = [] + +function source(): string { + return readFileSync(LOGIN_SCREEN, 'utf8') +} + +function warnsWithinWindow(text: string, entry: string): boolean { + const lines = text.split('\n') + const at = lines.findIndex(l => l.includes(`authApi.${entry}(`)) + if (at < 0) return false + return lines.slice(at, at + WINDOW).some(l => l.includes('warnIfNothingCarried(')) +} + +describe('guest-merge warning on every mobile sign-in path', () => { + it('the login screen still calls all four merge entry points', () => { + // A rename would make every assertion below pass vacuously, which is the failure mode of a + // grep-based test. + const text = source() + for (const entry of ENTRY_POINTS) { + expect(text, `authApi.${entry}( not found in login.tsx`).toContain(`authApi.${entry}(`) + } + expect(text).toContain('warnIfNothingCarried') + }) + + it.each(ENTRY_POINTS.filter(e => !KNOWN_GAPS.some(g => g.entry === e)))( + '%s warns when the server reports a skipped merge', + entry => { + expect(warnsWithinWindow(source(), entry)).toBe(true) + }, + ) + + it.each(KNOWN_GAPS)( + 'KNOWN GAP: $entry does not warn — $why', + ({ entry }) => { + // Characterization. When the call is added, this expectation flips to `true` and the entry + // moves out of KNOWN_GAPS into the list above. + expect(warnsWithinWindow(source(), entry)).toBe(false) + }, + ) + + it('warnIfNothingCarried is a no-op when nothing was skipped', () => { + // The field is null on the ordinary path and the guard must be the first thing in the function, + // or every successful sign-in shows an error toast. + const text = source() + const at = text.indexOf('const warnIfNothingCarried') + expect(at).toBeGreaterThan(-1) + const body = text.slice(at, at + 400) + expect(body).toMatch(/if \(!skipped\) return/) + }) +}) diff --git a/backend/src/Application/Agents/TutorAgent.cs b/backend/src/Application/Agents/TutorAgent.cs index eb933c2af..040d4fa61 100644 --- a/backend/src/Application/Agents/TutorAgent.cs +++ b/backend/src/Application/Agents/TutorAgent.cs @@ -86,7 +86,12 @@ private static string BuildGoal(TutorInput input) sb.Append($"- card {f.WordId} → {verdict} ({f.ResponseTimeMs}ms)\n"); } sb.Append($"\nFetch the current due / weak cards again, drop cards they already got right, re-surface the ones they MISSED "); - sb.Append("with an easier context exercise (pull a real example sentence for a miss), and order up to "); + // "pull a real example sentence for a miss" lived here until 2026-09-12. It named + // get_example_sentence, which was deleted with the retrieval spine — #605 removed the + // instruction from AllowedTools and from SystemPrompt and missed this copy, which is + // sent on EVERY re-plan turn. The guard could not see it: it reflects over the tool + // list, not over the words. + sb.Append("with an easier context exercise, and order up to "); sb.Append($"{cap} items.\n"); } sb.Append("Only plan cards a tool returned — never invent a word or card id."); diff --git a/docs/changelog-archive/2026-H2.md b/docs/changelog-archive/2026-H2.md index 42631afd6..acfec7948 100644 --- a/docs/changelog-archive/2026-H2.md +++ b/docs/changelog-archive/2026-H2.md @@ -3,6 +3,41 @@ Full write-ups, newest first. The one-line index lives in [`../../CHANGELOG.md`](../../CHANGELOG.md); the incidents worth reading on their own are in [`../incidents/`](../incidents/README.md). + + +## QA — three regressions in code shipped the day before — backend, mobile, shared — 2026-09-12 + +An adversarial pass over the nine PRs of 2026-09-11. Everything asserted about those screens was an +assertion about a projection function, because no mobile component test exists to run. Three of the +findings were regressions introduced by those PRs — all three mine, all three in the half of the work +that had no human looking at it. + +**"Mark as finished" on the phone was a silent no-op on a slow clock.** The helper added to make web +and mobile write the *same* locator also sent `updatedAt: new Date().toISOString()` — a device clock. +The server reads that as a last-write-wins guard and, when it is not newer than the stored value, +answers **200 with the row untouched**. The shelf had already flipped optimistically, so the reader +saw "finished" and the server saw nothing. Web's `markAsRead` has never sent one. The guard is right +for a queued background sync and wrong for a button; the timestamp is gone, and a test asserts the +payload does not carry one while the ordinary progress write still does. + +**The Apple sign-in never warned about a dropped guest merge.** #609 wired `warnIfNothingCarried` +into three of the four merge entry points. An iOS reader whose guest row was not merged was landed in +a new account with no indication that their highlights, vocabulary and progress had stayed behind. +A literal-scanning test now covers all four, in the shape `capabilityLiterals` and `routeLiterals` +already use — because mobile has no screen test that could. + +**The Tutor kept ordering a tool that does not exist.** #605 removed `get_example_sentence` from the +allowed list and from the system prompt, and missed the copy in `BuildGoal` — sent on **every** +re-plan turn. The guard added with it could not see it: it reflects over a string array, while the +instruction is prose. + +So the guard learned to read. It now scans what the agent *says* for anything shaped like a tool name +and requires it to be allowed. Two false starts are worth recording: matching against *registered* +tools left the mutation green (a deleted tool is registered nowhere, which is exactly the case that +hurts), and a naive scan failed on a correct file because these prompts wrap at 110 columns and a +name lands across a concatenation. It now glues adjacent literals, and both mutations — the plain one +and the split one — turn it red. + ## Guests — the reader is told when their work did not come across — web, mobile, shared — 2026-09-11 diff --git a/packages/shared/src/api/markProgressFinished.test.ts b/packages/shared/src/api/markProgressFinished.test.ts new file mode 100644 index 000000000..53135f5a5 --- /dev/null +++ b/packages/shared/src/api/markProgressFinished.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { initApi } from './client' +import { markProgressFinished, updateProgress } from './readingProgress' + +/** + * What "mark as finished" puts on the wire. + * + *

The server treats `updatedAt` as a last-write-wins guard: a timestamp that is not newer than + * the stored one makes the whole write a **no-op, answered 200 with the row untouched** + * (`UserDataEndpoints.UpsertProgress`). That is right for a queued background sync and wrong for a + * button: the reader taps "mark as finished", the shelf flips optimistically, and on a device whose + * clock runs a minute behind the server — ordinary, and invisible — nothing is saved and nothing + * says so.

+ * + *

Found by an adversarial QA pass on 2026-09-12, in code shipped the day before. Web's + * `markAsRead` never sent one; the mobile helper introduced to make the two clients agree quietly + * introduced this asymmetry instead.

+ */ +describe('markProgressFinished', () => { + const realFetch = globalThis.fetch + let fetchMock: ReturnType + + beforeEach(() => { + fetchMock = vi.fn(async () => ({ ok: true, status: 204, text: async () => '' }) as unknown as Response) + globalThis.fetch = fetchMock as unknown as typeof fetch + initApi({ + baseUrl: 'https://api.test', + getAccessToken: async () => 'token', + onUnauthorized: async () => null, + }) + }) + + afterEach(() => { + globalThis.fetch = realFetch + vi.restoreAllMocks() + }) + + const bodyOf = () => JSON.parse((fetchMock.mock.calls[0][1] as RequestInit).body as string) + + it('sends no updatedAt — a device clock must not be able to veto a deliberate tap', async () => { + await markProgressFinished('e1', { chapterId: 'c1', finished: true }) + + expect(bodyOf()).not.toHaveProperty('updatedAt') + }) + + it('finishes with the end sentinel and a whole-book percent', async () => { + await markProgressFinished('e1', { chapterId: 'c1', finished: true }) + + expect(bodyOf()).toMatchObject({ + chapterId: 'c1', + locator: '{"type":"end"}', + percent: 1, + percentUnit: 'book', + }) + }) + + it('un-finishes with the start sentinel and zero', async () => { + await markProgressFinished('e1', { chapterId: 'c1', finished: false }) + + expect(bodyOf()).toMatchObject({ locator: '{"type":"start"}', percent: 0 }) + }) + + it('leaves the ordinary progress write alone — it IS a queued sync and keeps its timestamp', async () => { + // The guard exists for this caller: reader progress is flushed from a queue, offline and out of + // order, where an older write must not overwrite a newer one. + await updateProgress('e1', { chapterId: 'c1', chapterSlug: 'ch-1', progress: 0.4 }) + + expect(bodyOf()).toHaveProperty('updatedAt') + }) +}) diff --git a/packages/shared/src/api/readingProgress.ts b/packages/shared/src/api/readingProgress.ts index 363c9f56e..1743ba874 100644 --- a/packages/shared/src/api/readingProgress.ts +++ b/packages/shared/src/api/readingProgress.ts @@ -66,6 +66,14 @@ export function markProgressFinished( locator: data.finished ? PROGRESS_LOCATOR_END : PROGRESS_LOCATOR_START, percent: data.finished ? 1 : 0, percentUnit: PERCENT_UNIT_BOOK, - updatedAt: new Date().toISOString(), + // NO `updatedAt`, deliberately — and web's markAsRead has never sent one either. + // + // The server treats it as a last-write-wins guard: a timestamp that is not newer than the + // stored one makes the whole write a no-op, answered 200 with the row untouched + // (UserDataEndpoints.UpsertProgress). That guard is for a queued background sync, where an old + // queued write must not overwrite a fresh one. This is neither queued nor background: the + // reader just tapped "mark as finished" and the shelf has already flipped optimistically. On a + // device whose clock runs a minute behind the server — ordinary, and invisible to the reader — + // the tap did nothing and said it worked. })) } diff --git a/tests/TextStack.Ai.Mcp.Tests/SetBookProgressPercentTests.cs b/tests/TextStack.Ai.Mcp.Tests/SetBookProgressPercentTests.cs new file mode 100644 index 000000000..ad55fb801 --- /dev/null +++ b/tests/TextStack.Ai.Mcp.Tests/SetBookProgressPercentTests.cs @@ -0,0 +1,165 @@ +using System.Text.Json; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; + +namespace TextStack.Ai.Mcp.Tests; + +/// +/// What number set_book_progress actually writes into ReadingProgress.Percent. +/// +/// That column is the ONE canonical answer to "how far through is this book" — every shelf, +/// card and detail screen reads it, and the whole percentUnit contract exists to keep a +/// client from putting a differently-derived number in it (see +/// Application.ReadingTracking.ProgressUnit). The app derives it word-weighted: +/// computeBookProgress in packages/shared/src/reader/bookProgress.ts sums the word +/// counts of the chapters before the current one over the book's total. +/// +/// McpToolCatalog.AfterFinishing derives it differently: chapters-done over +/// chapters-total. Both declare percentUnit: "book", so the server trusts both. On a book +/// whose chapters are all the same length the two agree and nothing shows — which is why +/// McpOverTheWireTests, whose Dracula fixture has two chapters of 4200 and 3900 words, cannot +/// see this. Real uploads are not that shape: front matter is many short chapters and the body is +/// one or two long ones. +/// +/// The first test is characterization of a defect. It asserts the chapter-count number +/// the bridge sends today and states, in the same test, the word-weighted number the reader's own +/// app would have computed for the same position. When the bridge is fixed to weight by +/// wordCount (which get_book and get_my_book both already return, alongside +/// totalWordCount), the expectations swap. +/// +public class SetBookProgressPercentTests : IAsyncLifetime +{ + private McpServerHarness _harness = null!; + + public async ValueTask InitializeAsync() => + _harness = await McpServerHarness.StartAsync(TestContext.Current.CancellationToken); + + public async ValueTask DisposeAsync() => await _harness.DisposeAsync(); + + private CancellationToken Ct => TestContext.Current.CancellationToken; + + private static Dictionary Args(params (string Key, object? Value)[] pairs) + { + var d = new Dictionary(StringComparer.Ordinal); + foreach (var (k, v) in pairs) d[k] = v; + return d; + } + + private static string TextOf(CallToolResult r) => ((TextContentBlock)r.Content[0]).Text; + + private static void AssertOk(CallToolResult r) => Assert.False(r.IsError == true, TextOf(r)); + + /// The app's formula, for the position "just finished chapter ". + private static double WordWeighted(int[] wordCounts, int finishedIndexInclusive) + { + var total = wordCounts.Sum(); + var done = wordCounts.Take(finishedIndexInclusive + 1).Sum(); + return (double)done / total; + } + + /// + /// DEFECT (characterized). The reader finished the preface — the 5th of six chapters, and + /// 4% of the words. The bridge records 83%. + /// + [Fact] + public async Task SetBookProgress_ShortFrontMatter_WritesChaptersDoneNotWordsRead() + { + await using var client = await _harness.ConnectAsync(McpServerHarness.TestJwt, Ct); + + var result = await client.CallToolAsync( + "set_book_progress", + Args(("slug", StubBackend.FrontMatterSlug), ("chapterSlug", "preface"))!, + cancellationToken: Ct); + + AssertOk(result); + + var body = JsonDocument.Parse(_harness.Stub.Last("set_edition_progress")!.Body).RootElement; + var sent = body.GetProperty("percent").GetDouble(); + + // 5 of 6 chapters. + Assert.Equal(5d / 6d, sent, 6); + Assert.Equal("book", body.GetProperty("percentUnit").GetString()); + + // What the reader's own app would have stored for the same position: 1888 of 45204 words. + var honest = WordWeighted(StubBackend.FrontMatterWordCounts, finishedIndexInclusive: 4); + Assert.True(honest < 0.05, $"fixture check: expected the preface to be <5% of the book, got {honest:P1}"); + + // The two disagree by more than 75 points on an ordinary non-fiction shape, and the larger + // one is the one that gets stored — the smaller is only ever recomputed inside the reader. + Assert.True(sent - honest > 0.75, + $"expected a large divergence to characterize; sent {sent:P1}, word-weighted {honest:P1}"); + } + + /// + /// The same call on a book whose chapters ARE evenly sized: the two formulas agree, which is + /// exactly why the existing Dracula fixture cannot show the problem. Kept so a future reader can + /// see that the fixture, not the code, was what made this invisible. + /// + [Fact] + public async Task SetBookProgress_EvenChapters_ChapterCountAndWordWeightedAgree() + { + await using var client = await _harness.ConnectAsync(McpServerHarness.TestJwt, Ct); + + var result = await client.CallToolAsync( + "set_book_progress", + Args(("slug", "dracula"), ("chapterSlug", "ch-1"))!, + cancellationToken: Ct); + + AssertOk(result); + + var sent = JsonDocument.Parse(_harness.Stub.Last("set_edition_progress")!.Body) + .RootElement.GetProperty("percent").GetDouble(); + var honest = WordWeighted([4200, 3900], finishedIndexInclusive: 0); + + Assert.Equal(0.5, sent, 6); + Assert.True(Math.Abs(sent - honest) < 0.02, + $"fixture check: the two formulas should be within 2 points here; {sent:P1} vs {honest:P1}"); + } + + /// + /// Finishing the last chapter must round to exactly 1.0 whichever formula is used — the server + /// turns >= 0.99 into CompletedAt, and a book the reader said they finished has + /// to be finished. + /// + [Fact] + public async Task SetBookProgress_LastChapterOfALopsidedBook_IsExactlyOne() + { + await using var client = await _harness.ConnectAsync(McpServerHarness.TestJwt, Ct); + + var result = await client.CallToolAsync( + "set_book_progress", + Args(("slug", StubBackend.FrontMatterSlug), ("chapterSlug", "the-book-itself"))!, + cancellationToken: Ct); + + AssertOk(result); + var json = JsonDocument.Parse(TextOf(result)).RootElement; + Assert.True(json.GetProperty("bookFinished").GetBoolean()); + + var body = JsonDocument.Parse(_harness.Stub.Last("set_edition_progress")!.Body).RootElement; + Assert.Equal(1d, body.GetProperty("percent").GetDouble()); + Assert.Equal("""{"type":"end"}""", body.GetProperty("locator").GetString()); + } + + /// + /// The inverse of the front-matter case, and the one that costs the reader most: a book whose + /// LAST chapter is a one-page acknowledgements section. Finishing the real final chapter is 99% + /// of the words and the bridge records it as 83%, so the book does not become finished. + /// + [Fact] + public async Task SetBookProgress_SecondToLastChapter_DoesNotFinishABookTheReaderHasEffectivelyFinished() + { + await using var client = await _harness.ConnectAsync(McpServerHarness.TestJwt, Ct); + + var result = await client.CallToolAsync( + "set_book_progress", + Args(("slug", StubBackend.FrontMatterSlug), ("chapterSlug", "preface"))!, + cancellationToken: Ct); + + AssertOk(result); + Assert.False(JsonDocument.Parse(TextOf(result)).RootElement.GetProperty("bookFinished").GetBoolean()); + + var sent = JsonDocument.Parse(_harness.Stub.Last("set_edition_progress")!.Body) + .RootElement.GetProperty("percent").GetDouble(); + Assert.True(sent < 0.99, "chapters-done never reaches the completion threshold before the last chapter"); + } +} diff --git a/tests/TextStack.Ai.Mcp.Tests/StubBackend.cs b/tests/TextStack.Ai.Mcp.Tests/StubBackend.cs index 66d4833db..d9b47d55a 100644 --- a/tests/TextStack.Ai.Mcp.Tests/StubBackend.cs +++ b/tests/TextStack.Ai.Mcp.Tests/StubBackend.cs @@ -98,7 +98,14 @@ private void MapRoutes() _app.MapGet("/books/{slug}", async ctx => { await RecordAsync("get_book", ctx); - await WriteJsonAsync(ctx, BookDetailBody); + // One extra canned book, for the percentage only. Dracula's two chapters are almost the + // same length, so chapters-done/total and a word-weighted fraction agree to within two + // points there — a fixture that cannot tell the two formulas apart. + var slug = (string?)ctx.Request.RouteValues["slug"]; + await WriteJsonAsync(ctx, + string.Equals(slug, FrontMatterSlug, StringComparison.Ordinal) + ? FrontMatterBookDetailBody + : BookDetailBody); }); // GET /books/{slug}/chapters/{chapterSlug} → ChapterDto (html, prev/next). @@ -342,6 +349,38 @@ public async ValueTask DisposeAsync() } """; + /// + /// A book whose chapters are NOT the same size: five short front-matter chapters and one long + /// body chapter, which is the ordinary shape of a non-fiction upload. It exists so the + /// percentage set_book_progress writes can be compared against the word-weighted one the + /// app's own computeBookProgress produces for the same position. + /// + public const string FrontMatterSlug = "front-matter"; + + private const string FrontMatterBookDetailBody = + """ + { + "id": "33333333-3333-3333-3333-333333333333", + "slug": "front-matter", + "title": "A Book With Front Matter", + "language": "en", + "description": "Five short chapters and one long one.", + "authors": [{ "id": "1", "slug": "an", "name": "A. N. Other", "role": "author" }], + "genres": [{ "id": "2", "slug": "nonfiction", "name": "Non-fiction" }], + "chapters": [ + { "id": "10000000-0000-0000-0000-000000000001", "chapterNumber": 1, "slug": "title-page", "title": "Title Page", "wordCount": 14 }, + { "id": "10000000-0000-0000-0000-000000000002", "chapterNumber": 2, "slug": "copyright", "title": "Copyright", "wordCount": 333 }, + { "id": "10000000-0000-0000-0000-000000000003", "chapterNumber": 3, "slug": "dedication", "title": "Dedication", "wordCount": 125 }, + { "id": "10000000-0000-0000-0000-000000000004", "chapterNumber": 4, "slug": "contents", "title": "Contents", "wordCount": 232 }, + { "id": "10000000-0000-0000-0000-000000000005", "chapterNumber": 5, "slug": "preface", "title": "Preface", "wordCount": 1184 }, + { "id": "10000000-0000-0000-0000-000000000006", "chapterNumber": 6, "slug": "the-book-itself", "title": "The Book Itself", "wordCount": 43310 } + ] + } + """; + + /// Word counts of , in order. + public static readonly int[] FrontMatterWordCounts = [14, 333, 125, 232, 1184, 43310]; + private const string ChapterBody = """ { diff --git a/tests/TextStack.IntegrationTests/MarkFinishedClientClockTests.cs b/tests/TextStack.IntegrationTests/MarkFinishedClientClockTests.cs new file mode 100644 index 000000000..a3e44c80d --- /dev/null +++ b/tests/TextStack.IntegrationTests/MarkFinishedClientClockTests.cs @@ -0,0 +1,208 @@ +using System.Net.Http.Json; +using System.Text.Json; + +namespace TextStack.IntegrationTests; + +/// +/// "Mark finished" on a catalog book, and the client clock it is decided by. +/// +/// #602 gave mobile markProgressFinished (packages/shared/src/api/readingProgress.ts) +/// so that the phone and the browser write the same sentinel for the same action. They do. What they +/// do not share is the stale-write guard: markProgressFinished sends +/// updatedAt: new Date().toISOString() — a DEVICE clock — while web's markAsRead +/// (apps/web/src/api/auth.ts) sends none. +/// +/// UserDataEndpoints.UpsertProgress compares that client timestamp against the row's +/// server-written UpdatedAt and, when it is not newer, returns 200 with the row +/// untouched. So on a device whose clock is behind the server, tapping "Mark finished" is a +/// no-op that reports success — and useBookActions has already optimistically flipped the +/// shelf to finished, so the UI and the server disagree until the next fetch. +/// +/// The same reasoning has already been applied once, in the other direction: +/// UserBookService.UpsertProgressAsync DELETED its client-clock gate for exactly this failure +/// ("a device clock a second behind the server's was silently dropped"). The catalog path kept it. +/// +/// These two tests are characterization. They assert what the server does today, and +/// they are named for the fact that it is wrong. When the guard is fixed — by ignoring +/// updatedAt for a sentinel write, by comparing client clocks only against client clocks, or +/// by answering 409 instead of a silent 200 — both should be inverted, and the first one's name is +/// the assertion the fix should make true. +/// +/// Requires docker compose up + ENABLE_TEST_AUTH=true; skips rather than fails otherwise. +/// +public class MarkFinishedClientClockTests(LiveApiFixture fixture, AuthenticatedApiFixture auth) + : IClassFixture, IClassFixture +{ + private const string EndOfBook = """{"type":"end"}"""; + private const string StartOfChapter = """{"type":"start"}"""; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// + /// Deliberately NOT the first seeded edition. ProgressUnitEndpointTests takes + /// /books?limit=1 and DELETEs that row's progress for the same test user; xUnit runs the + /// two classes in parallel, so sharing an edition makes both flaky for reasons that have nothing + /// to do with what either is testing. + /// + private const int EditionIndex = 1; + + private async Task<(Guid EditionId, Guid ChapterId)?> FindSeededChapterAsync() + { + var listResp = await fixture.Client.SendAsync(fixture.CreateRequest(HttpMethod.Get, "/books?limit=5"), Ct); + if (!listResp.IsSuccessStatusCode) return null; + + var list = await listResp.Content.ReadFromJsonAsync(cancellationToken: Ct); + if (!list.TryGetProperty("items", out var items) || items.GetArrayLength() <= EditionIndex) return null; + + var bookResp = await fixture.Client.SendAsync( + fixture.CreateRequest(HttpMethod.Get, $"/books/{items[EditionIndex].GetProperty("slug").GetString()}"), Ct); + if (!bookResp.IsSuccessStatusCode) return null; + + var book = await bookResp.Content.ReadFromJsonAsync(cancellationToken: Ct); + if (!book.TryGetProperty("chapters", out var chapters) || chapters.GetArrayLength() == 0) return null; + + return (book.GetProperty("id").GetGuid(), chapters[0].GetProperty("id").GetGuid()); + } + + private async Task PutAsync(Guid editionId, object body) + { + var req = auth.CreateRequest(HttpMethod.Put, $"/me/progress/{editionId}"); + req.Content = JsonContent.Create(body); + return await auth.Client.SendAsync(req, Ct); + } + + private async Task ReadAsync(Guid editionId) + { + var read = await auth.Client.SendAsync( + auth.CreateRequest(HttpMethod.Get, $"/me/progress/{editionId}"), Ct); + return await read.Content.ReadFromJsonAsync(cancellationToken: Ct); + } + + /// + /// The payload markProgressFinished(editionId, { chapterId, finished: true }) puts on the + /// wire, with the device clock as the caller supplies it. + /// + private static object MarkFinishedBody(Guid chapterId, DateTimeOffset deviceNow) => new + { + chapterId, + locator = EndOfBook, + percent = 1.0, + percentUnit = "book", + updatedAt = deviceNow.ToString("O"), + }; + + /// + /// DEFECT (characterized). Device clock 60 seconds behind the server: the book is not + /// finished, and the caller is told 200. + /// + [Fact] + public async Task MarkFinished_DeviceClockBehindTheServer_Returns200AndFinishesNothing() + { + var seeded = await FindSeededChapterAsync(); + Assert.SkipWhen(seeded is null, "no seeded edition with chapters"); + Assert.SkipWhen(!auth.IsAuthenticated, "test auth unavailable"); + var (editionId, chapterId) = seeded!.Value; + + // A reader mid-book: a fresh row whose UpdatedAt is the server's "now". + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + var seed = await PutAsync(editionId, new + { + chapterId, + locator = StartOfChapter, + percent = 0.2, + percentUnit = "book", + }); + Assert.SkipWhen(IntegrationSkip.Unavailable(seed), "endpoint unavailable"); + Assert.True(seed.IsSuccessStatusCode, await seed.Content.ReadAsStringAsync(Ct)); + + var put = await PutAsync(editionId, MarkFinishedBody(chapterId, DateTimeOffset.UtcNow.AddSeconds(-60))); + Assert.SkipWhen(IntegrationSkip.Unavailable(put), "endpoint unavailable"); + + // Success, as far as any client can tell — the body is ignored by `authFetch`. + Assert.True(put.IsSuccessStatusCode, await put.Content.ReadAsStringAsync(Ct)); + + var row = await ReadAsync(editionId); + // …and nothing moved. This is the line to invert when the guard is fixed: + // the book SHOULD be finished here. + Assert.Equal(0.2, row.GetProperty("percent").GetDouble()); + Assert.Equal(StartOfChapter, row.GetProperty("locator").GetString()); + Assert.Equal(JsonValueKind.Null, row.GetProperty("completedAt").ValueKind); + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + } + + /// + /// The control: the same request with a clock that is not behind does finish the book. Without + /// this, "mark finished never works" would also pass the test above. + /// + [Fact] + public async Task MarkFinished_DeviceClockAheadOfTheServer_FinishesTheBook() + { + var seeded = await FindSeededChapterAsync(); + Assert.SkipWhen(seeded is null, "no seeded edition with chapters"); + Assert.SkipWhen(!auth.IsAuthenticated, "test auth unavailable"); + var (editionId, chapterId) = seeded!.Value; + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + var seed = await PutAsync(editionId, new + { + chapterId, + locator = StartOfChapter, + percent = 0.2, + percentUnit = "book", + }); + Assert.SkipWhen(IntegrationSkip.Unavailable(seed), "endpoint unavailable"); + Assert.True(seed.IsSuccessStatusCode, await seed.Content.ReadAsStringAsync(Ct)); + + var put = await PutAsync(editionId, MarkFinishedBody(chapterId, DateTimeOffset.UtcNow.AddMinutes(1))); + Assert.SkipWhen(IntegrationSkip.Unavailable(put), "endpoint unavailable"); + Assert.True(put.IsSuccessStatusCode, await put.Content.ReadAsStringAsync(Ct)); + + var row = await ReadAsync(editionId); + Assert.Equal(1.0, row.GetProperty("percent").GetDouble()); + Assert.Equal(EndOfBook, row.GetProperty("locator").GetString()); + Assert.Equal(JsonValueKind.String, row.GetProperty("completedAt").ValueKind); + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + } + + /// + /// Web's shape of the same action — no updatedAt at all — is unconditional. Pinned so the + /// asymmetry between the two clients is visible in the suite rather than only in review. + /// + [Fact] + public async Task MarkFinished_WithNoClientTimestampAtAll_AlwaysFinishesTheBook() + { + var seeded = await FindSeededChapterAsync(); + Assert.SkipWhen(seeded is null, "no seeded edition with chapters"); + Assert.SkipWhen(!auth.IsAuthenticated, "test auth unavailable"); + var (editionId, chapterId) = seeded!.Value; + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + var seed = await PutAsync(editionId, new + { + chapterId, + locator = StartOfChapter, + percent = 0.2, + percentUnit = "book", + }); + Assert.SkipWhen(IntegrationSkip.Unavailable(seed), "endpoint unavailable"); + Assert.True(seed.IsSuccessStatusCode, await seed.Content.ReadAsStringAsync(Ct)); + + var put = await PutAsync(editionId, new + { + chapterId, + locator = EndOfBook, + percent = 1.0, + percentUnit = "book", + }); + Assert.SkipWhen(IntegrationSkip.Unavailable(put), "endpoint unavailable"); + Assert.True(put.IsSuccessStatusCode, await put.Content.ReadAsStringAsync(Ct)); + + var row = await ReadAsync(editionId); + Assert.Equal(1.0, row.GetProperty("percent").GetDouble()); + Assert.Equal(JsonValueKind.String, row.GetProperty("completedAt").ValueKind); + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + } +} diff --git a/tests/TextStack.IntegrationTests/ShelfAfterAssistantProgressTests.cs b/tests/TextStack.IntegrationTests/ShelfAfterAssistantProgressTests.cs new file mode 100644 index 000000000..306defa0f --- /dev/null +++ b/tests/TextStack.IntegrationTests/ShelfAfterAssistantProgressTests.cs @@ -0,0 +1,157 @@ +using System.Net.Http.Json; +using System.Text.Json; + +namespace TextStack.IntegrationTests; + +/// +/// What get_my_reading can see after set_book_progress has written. +/// +/// #601 gave the assistant three tools and told it, in get_my_reading's own description, +/// "Call this FIRST when you do not already have a bookId or editionId — nothing else here can find a +/// book without one." That tool is GET /me/library/shelves. Its saved-book shelf is an INNER +/// JOIN on user_libraries: a catalog book the reader has never explicitly saved is not on it, +/// whatever its progress says. +/// +/// The reader's own app hides this because ReaderPage auto-adds a book to the library at +/// 1% (apps/web/src/pages/ReaderPage.tsx). set_book_progress does not: it writes +/// PUT /me/progress/{editionId} and nothing else. So an assistant that records "you finished +/// chapter 3 of Dracula" for a book the reader has read only outside the app cannot find that book +/// again on its next call — get_book_progress still answers, but only if the model still +/// happens to hold the editionId. +/// +/// Characterization. The first test asserts the hole. When set_book_progress +/// starts putting the book on the shelf (a POST /me/library/{editionId} alongside the +/// progress write), it should assert presence instead. +/// +/// Requires docker compose up + ENABLE_TEST_AUTH=true; skips rather than fails otherwise. +/// +public class ShelfAfterAssistantProgressTests(LiveApiFixture fixture, AuthenticatedApiFixture auth) + : IClassFixture, IClassFixture +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// + /// A third edition, for the same reason MarkFinishedClientClockTests takes the second: + /// these classes run in parallel against one test user, and each rewrites the progress row of + /// the edition it picks. Index 0 belongs to ProgressUnitEndpointTests. + /// + private const int EditionIndex = 2; + + private async Task<(Guid EditionId, Guid ChapterId)?> FindSeededChapterAsync() + { + var listResp = await fixture.Client.SendAsync(fixture.CreateRequest(HttpMethod.Get, "/books?limit=5"), Ct); + if (!listResp.IsSuccessStatusCode) return null; + + var list = await listResp.Content.ReadFromJsonAsync(cancellationToken: Ct); + if (!list.TryGetProperty("items", out var items) || items.GetArrayLength() <= EditionIndex) return null; + + var bookResp = await fixture.Client.SendAsync( + fixture.CreateRequest(HttpMethod.Get, $"/books/{items[EditionIndex].GetProperty("slug").GetString()}"), Ct); + if (!bookResp.IsSuccessStatusCode) return null; + + var book = await bookResp.Content.ReadFromJsonAsync(cancellationToken: Ct); + if (!book.TryGetProperty("chapters", out var chapters) || chapters.GetArrayLength() == 0) return null; + + return (book.GetProperty("id").GetGuid(), chapters[0].GetProperty("id").GetGuid()); + } + + /// Every edition id on the "continue reading" + "finished this month" shelves. + private async Task> ShelfEditionIdsAsync() + { + var resp = await auth.Client.SendAsync( + auth.CreateRequest(HttpMethod.Get, "/me/library/shelves"), Ct); + Assert.SkipWhen(IntegrationSkip.Unavailable(resp), "/me/library/shelves unavailable"); + resp.EnsureSuccessStatusCode(); + + var body = await resp.Content.ReadFromJsonAsync(cancellationToken: Ct); + var ids = new HashSet(); + foreach (var shelf in new[] { "continueReading", "finishedThisMonth" }) + { + if (!body.TryGetProperty(shelf, out var arr) || arr.ValueKind != JsonValueKind.Array) continue; + foreach (var item in arr.EnumerateArray()) + { + if (item.TryGetProperty("type", out var t) && t.GetString() == "savedbook" + && item.TryGetProperty("id", out var id)) + ids.Add(id.GetGuid()); + } + } + return ids; + } + + /// + /// DEFECT (characterized). The exact write set_book_progress makes for a catalog + /// book, on a book the reader has not saved: stored, readable by edition id, invisible to the + /// shelf the assistant is told to start from. + /// + [Fact] + public async Task ProgressWrittenWithoutSavingTheBook_IsInvisibleToTheShelf() + { + var seeded = await FindSeededChapterAsync(); + Assert.SkipWhen(seeded is null, "no seeded edition with chapters"); + Assert.SkipWhen(!auth.IsAuthenticated, "test auth unavailable"); + var (editionId, chapterId) = seeded!.Value; + + // Start from "not saved, no progress". + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/library/{editionId}"), Ct); + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + + var put = auth.CreateRequest(HttpMethod.Put, $"/me/progress/{editionId}"); + put.Content = JsonContent.Create(new + { + chapterId, + locator = """{"type":"start"}""", + percent = 0.25, + percentUnit = "book", + }); + var wrote = await auth.Client.SendAsync(put, Ct); + Assert.SkipWhen(IntegrationSkip.Unavailable(wrote), "/me/progress unavailable"); + Assert.True(wrote.IsSuccessStatusCode, await wrote.Content.ReadAsStringAsync(Ct)); + + // The position is genuinely there — get_book_progress answers from this. + var read = await auth.Client.SendAsync( + auth.CreateRequest(HttpMethod.Get, $"/me/progress/{editionId}"), Ct); + var row = await read.Content.ReadFromJsonAsync(cancellationToken: Ct); + Assert.Equal(0.25, row.GetProperty("percent").GetDouble()); + + // …and the shelf the assistant is told to call FIRST does not list it. + Assert.DoesNotContain(editionId, await ShelfEditionIdsAsync()); + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + } + + /// + /// The control: the same progress, plus the library row the reader's own app would have created + /// at 1%, and the book appears. Without this, "the shelf is always empty" would also pass above. + /// + [Fact] + public async Task SameProgress_WithTheLibraryRowTheAppWouldHaveCreated_IsOnTheShelf() + { + var seeded = await FindSeededChapterAsync(); + Assert.SkipWhen(seeded is null, "no seeded edition with chapters"); + Assert.SkipWhen(!auth.IsAuthenticated, "test auth unavailable"); + var (editionId, chapterId) = seeded!.Value; + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + + var put = auth.CreateRequest(HttpMethod.Put, $"/me/progress/{editionId}"); + put.Content = JsonContent.Create(new + { + chapterId, + locator = """{"type":"start"}""", + percent = 0.25, + percentUnit = "book", + }); + var wrote = await auth.Client.SendAsync(put, Ct); + Assert.SkipWhen(IntegrationSkip.Unavailable(wrote), "/me/progress unavailable"); + Assert.True(wrote.IsSuccessStatusCode, await wrote.Content.ReadAsStringAsync(Ct)); + + var saved = await auth.Client.SendAsync( + auth.CreateRequest(HttpMethod.Post, $"/me/library/{editionId}"), Ct); + Assert.SkipWhen(IntegrationSkip.Unavailable(saved), "/me/library unavailable"); + + Assert.Contains(editionId, await ShelfEditionIdsAsync()); + + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/progress/{editionId}"), Ct); + await auth.Client.SendAsync(auth.CreateRequest(HttpMethod.Delete, $"/me/library/{editionId}"), Ct); + } +} diff --git a/tests/TextStack.UnitTests/AccessTokenIdentityTests.cs b/tests/TextStack.UnitTests/AccessTokenIdentityTests.cs new file mode 100644 index 000000000..6e5cee76d --- /dev/null +++ b/tests/TextStack.UnitTests/AccessTokenIdentityTests.cs @@ -0,0 +1,173 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Security.Claims; +using System.Text; +using Application.Auth; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + +namespace TextStack.UnitTests; + +/// +/// AuthService.ValidateAccessTokenIdentity — the seam #604 added so +/// GuestActivityMiddleware could learn, from the token, whether a request belongs to a guest. +/// +/// The middleware got a pure function and tests for its debounce +/// (GuestActivityDebounceTests). The part that was actually broken — deciding whether this is +/// a guest — got neither. It is the whole reason the middleware was inert for a year: the answer was +/// being read from HttpContext.User, which this API never populates. Reading it from the +/// right place is a claim that should be pinned, not re-verified by hand each time. +/// +/// No database is touched by the method under test, so IAppDbContext is not supplied. +/// +public class AccessTokenIdentityTests +{ + private const string Secret = "a-test-signing-key-long-enough-for-hmac-sha256-abcdefgh"; + private const string OtherSecret = "a-DIFFERENT-signing-key-long-enough-for-hmac-sha256-xyz"; + private const string Issuer = "textstack.app"; + + private static AuthService Service() => new( + db: null!, + jwtSettings: Options.Create(new JwtSettings { SecretKey = Secret, Issuer = Issuer }), + googleSettings: Options.Create(new GoogleSettings { ClientId = "unused.apps.googleusercontent.com" })); + + /// Mints a token the same shape AuthService.GenerateAccessToken does. + private static string Token( + Guid? userId, + bool guestClaim, + string? guestClaimValue = "true", + string issuer = Issuer, + string secret = Secret, + int expiresInMinutes = 60) + { + var claims = new List + { + new(ClaimTypes.Email, "reader@example.test"), + new(ClaimTypes.Name, "Reader"), + }; + if (userId is { } id) claims.Insert(0, new Claim(ClaimTypes.NameIdentifier, id.ToString())); + if (guestClaim) claims.Add(new Claim(AuthService.GuestClaimType, guestClaimValue!)); + + var key = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(secret)); + var token = new JwtSecurityToken( + issuer: issuer, + claims: claims, + expires: DateTime.UtcNow.AddMinutes(expiresInMinutes), + signingCredentials: new SigningCredentials(key, SecurityAlgorithms.HmacSha256)); + + return new JwtSecurityTokenHandler().WriteToken(token); + } + + [Fact] + public void ValidateAccessTokenIdentity_GuestToken_ReportsTheUserAndTheGuestFlag() + { + var id = Guid.NewGuid(); + + var (userId, isGuest) = Service().ValidateAccessTokenIdentity(Token(id, guestClaim: true)); + + Assert.Equal(id, userId); + Assert.True(isGuest); + } + + [Fact] + public void ValidateAccessTokenIdentity_AccountToken_ReportsTheUserAndNotAGuest() + { + // The middleware must not write LastActiveAt for an account: the column is only read by + // GuestCleanupWorker, and an UPDATE per request on every signed-in reader is pure cost. + var id = Guid.NewGuid(); + + var (userId, isGuest) = Service().ValidateAccessTokenIdentity(Token(id, guestClaim: false)); + + Assert.Equal(id, userId); + Assert.False(isGuest); + } + + [Fact] + public void ValidateAccessTokenIdentity_ExpiredGuestToken_IsNobody() + { + // The documented contract: (null, false) rather than an id, so no caller can mistake an + // expired token for an account. `ClockSkew = Zero`, so one minute in the past is expired. + var (userId, isGuest) = Service().ValidateAccessTokenIdentity( + Token(Guid.NewGuid(), guestClaim: true, expiresInMinutes: -1)); + + Assert.Null(userId); + Assert.False(isGuest); + } + + [Fact] + public void ValidateAccessTokenIdentity_ForeignSignature_IsNobody() + { + // Structurally a real guest token, signed with a key this deployment does not hold. + var (userId, isGuest) = Service().ValidateAccessTokenIdentity( + Token(Guid.NewGuid(), guestClaim: true, secret: OtherSecret)); + + Assert.Null(userId); + Assert.False(isGuest); + } + + [Fact] + public void ValidateAccessTokenIdentity_WrongIssuer_IsNobody() + { + var (userId, isGuest) = Service().ValidateAccessTokenIdentity( + Token(Guid.NewGuid(), guestClaim: true, issuer: "someone-elses.app")); + + Assert.Null(userId); + Assert.False(isGuest); + } + + [Fact] + public void ValidateAccessTokenIdentity_NoSubjectClaim_IsNobody() + { + var (userId, isGuest) = Service().ValidateAccessTokenIdentity( + Token(userId: null, guestClaim: true)); + + Assert.Null(userId); + Assert.False(isGuest); + } + + [Theory] + [InlineData("false")] + [InlineData("True")] // the claim is compared with ==, so casing matters + [InlineData("1")] + [InlineData("")] + public void ValidateAccessTokenIdentity_GuestClaimThatIsNotExactlyTrue_IsNotAGuest(string value) + { + var id = Guid.NewGuid(); + + var (userId, isGuest) = Service().ValidateAccessTokenIdentity( + Token(id, guestClaim: true, guestClaimValue: value)); + + Assert.Equal(id, userId); + Assert.False(isGuest); + } + + [Theory] + [InlineData("")] + [InlineData("not-a-jwt")] + [InlineData("tsk_a_connect_key_is_not_a_jwt")] // McpKeyAuthMiddleware's bearer reaches here too + [InlineData("a.b.c")] + public void ValidateAccessTokenIdentity_Garbage_IsNobodyAndDoesNotThrow(string token) + { + // A connect key is an account-level credential and never a guest's; it must fall through + // quietly rather than throw inside a middleware that runs on every request. + var (userId, isGuest) = Service().ValidateAccessTokenIdentity(token); + + Assert.Null(userId); + Assert.False(isGuest); + } + + [Fact] + public void ValidateAccessToken_AndTheIdentityOverload_CannotDisagree() + { + // One is defined in terms of the other precisely so the guest-activity path and every + // endpoint's GetUserId resolve the same user. Pinned so an optimisation cannot split them. + var service = Service(); + var id = Guid.NewGuid(); + var guest = Token(id, guestClaim: true); + var account = Token(id, guestClaim: false); + var dead = Token(id, guestClaim: true, expiresInMinutes: -1); + + Assert.Equal(service.ValidateAccessToken(guest), service.ValidateAccessTokenIdentity(guest).UserId); + Assert.Equal(service.ValidateAccessToken(account), service.ValidateAccessTokenIdentity(account).UserId); + Assert.Equal(service.ValidateAccessToken(dead), service.ValidateAccessTokenIdentity(dead).UserId); + } +} diff --git a/tests/TextStack.UnitTests/AgentAllowedToolsExistTests.cs b/tests/TextStack.UnitTests/AgentAllowedToolsExistTests.cs index 87a3ff200..ed66851b6 100644 --- a/tests/TextStack.UnitTests/AgentAllowedToolsExistTests.cs +++ b/tests/TextStack.UnitTests/AgentAllowedToolsExistTests.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Text.RegularExpressions; using System.Runtime.CompilerServices; using Application.Agents; using Application.Tools; @@ -80,4 +81,75 @@ public void AgentToolLists_FoundAtLeastOneAgent() // this kind of test. Renaming `AllTools` would do exactly that. Assert.NotEmpty(AgentToolLists()); } + + /// + /// Every tool an agent's PROSE names must be one it is allowed to call. + /// + /// The test above reflects over AllowedTools and cannot see a word. The Tutor's + /// BuildGoal went on telling the model to "pull a real example sentence for a miss" for a + /// day after get_example_sentence was deleted — on every re-plan turn, unobservably, + /// because the instruction is prose and the guard read a string array. + /// + /// It cannot be written as "is this a registered tool". The first version of this + /// test was, and a mutation putting the dead instruction back left it green: the name belongs to + /// no tool at all any more, which is exactly the case that hurts. So it matches the SHAPE of a + /// tool name — this repo's tools are all verb_noun in snake_case — and requires every one + /// it finds to be allowed. + /// + [Fact] + public void AgentProse_NamesNoToolTheAgentCannotCall() + { + var toolish = new Regex(@"\b(?:get|save|list|search|lookup|set)_[a-z][a-z_]{2,}\b", RegexOptions.Compiled); + var offenders = new List(); + + foreach (var agent in typeof(TutorAgent).Assembly.GetTypes() + .Where(t => t.Namespace == "Application.Agents" && !t.IsAbstract)) + { + var field = agent.GetField("AllTools", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public); + if (field?.GetValue(null) is not string[] allowed) continue; + + // Comments are stripped: this file's own explanation names the dead tool, and so do the + // notes left where an instruction was removed. What ships to the model is the strings. + var source = StripComments(File.ReadAllText(SourcePathFor(agent))); + + foreach (Match m in toolish.Matches(source)) + { + if (allowed.Contains(m.Value)) continue; + offenders.Add($"{agent.Name} says '{m.Value}', which is not in its AllowedTools"); + } + } + + Assert.True(offenders.Count == 0, string.Join("; ", offenders.Distinct()) + + ". An instruction naming a tool the model is not offered cannot be obeyed: it is paid " + + "for on every run and can only degrade the plan. Remove the words, or allow the tool."); + } + + /// + /// What the model actually receives, approximately: comments dropped, and adjacent string + /// literals glued. + /// + /// The gluing is not cosmetic. These prompts wrap at 110 columns, so a tool name lands + /// across a concatenation — "…get_weak_" + "vocabulary…" — and a naive scan reports + /// get_weak_, a name no list will ever contain. The first version of this helper did + /// exactly that and failed on a correct file. + /// + private static string StripComments(string source) + { + source = Regex.Replace(source, @"/\*.*?\*/", " ", RegexOptions.Singleline); + source = Regex.Replace(source, @"//[^\n]*", " "); + // "abc" + "def" -> "abcdef" (also across newlines, and past a verbatim @ prefix) + return Regex.Replace(source, "\"\\s*\\+\\s*@?\"", ""); + } + + /// The agent's own source file, found by name — these types are one-per-file. + private static string SourcePathFor(Type agent) + { + var root = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "../../../../..")); + var matches = Directory.GetFiles(Path.Combine(root, "backend/src/Application/Agents"), + agent.Name + ".cs", SearchOption.AllDirectories); + Assert.True(matches.Length == 1, + $"expected exactly one source file for {agent.Name}, found {matches.Length} — this test " + + "reads the file to see what the agent SAYS, so it must find it"); + return matches[0]; + } } diff --git a/tests/TextStack.UnitTests/McOptionsCascadeTests.cs b/tests/TextStack.UnitTests/McOptionsCascadeTests.cs new file mode 100644 index 000000000..099145c2a --- /dev/null +++ b/tests/TextStack.UnitTests/McOptionsCascadeTests.cs @@ -0,0 +1,143 @@ +using TextStack.Vocabulary; +using TextStack.Vocabulary.Contracts; + +namespace TextStack.UnitTests; + +/// +/// Adversarial coverage for — the edges the extraction in #606 did not pin. +/// +/// McOptionsTests covers the happy cascade (LLM → learner's own words → hardcoded list) +/// and that four options always come back with the answer among them. What it does not cover is what +/// the LAST resort actually contains, and that is where the card stops being a question. +/// +/// Two tests here are characterization: they assert what the code does today, and they +/// are named for the fact that what it does today is wrong. Inverting them is part of the fix, not a +/// separate chore. They are written this way rather than as red tests so the suite stays honest about +/// its own state — see the docblock on each. +/// +public class McOptionsCascadeTests +{ + private static string Json(params string[] words) => System.Text.Json.JsonSerializer.Serialize(words); + + /// + /// DEFECT (characterized, not fixed). DistractorWords.ForLanguage has entries for + /// en/de/fr/es and returns English for everything else. So a learner saving a Ukrainian, + /// Italian or Polish word — with no LLM distractors yet (they are generated fire-and-forget + /// AFTER the save) and an empty personal pool (their first words) — is shown their word next to + /// three English nouns. The answer is the only option in the right alphabet: the card can be + /// passed without knowing the word, and passing it advances the SRS stage. + /// + /// #606 widened the blast radius: the Tutor now builds its recognition and + /// context options through this same method, so a tutor session inherits it. + /// + /// When fixed — by returning an empty list for an unknown language and letting the caller + /// decide, or by refusing to build an MC card at all — this assertion should become + /// Assert.DoesNotContain. + /// + [Fact] + public void Build_LanguageWithNoDistractorList_FillsWithEnglish_SoTheAnswerIsTheOnlyNonEnglishOption() + { + var (options, correct) = McOptions.Build("осяжний", "uk", distractorsJson: null, pool: []); + + Assert.Equal(McOptions.Choices, options.Count); + Assert.Equal("осяжний", options[correct]); + + var fillers = options.Where((_, i) => i != correct).ToList(); + Assert.All(fillers, f => Assert.Contains(f, DistractorWords.English)); + } + + /// + /// The same shape stated as the property that actually matters, so the fix has something to aim + /// at: every option should be in the card's language. Characterized as FALSE today. + /// + [Fact] + public void Build_LanguageWithNoDistractorList_DoesNotProduceASameLanguageCard() + { + var (options, correct) = McOptions.Build("ubiquitario", "it", distractorsJson: null, pool: []); + + // Every distractor is drawn from the English list; none of them is Italian. + var distractors = options.Where((_, i) => i != correct); + Assert.All(distractors, d => Assert.Contains(d, DistractorWords.English)); + } + + /// + /// The LLM gate is Count >= 3, and the answer-filter runs AFTER it. Three distractors + /// that are all the answer in different cases pass the gate and contribute nothing — the cascade + /// must still fill the card rather than emit a two- or three-option one. + /// + [Fact] + public void Build_LlmDistractorsAreAllTheAnswer_StillReturnsFourOptions() + { + var (options, correct) = McOptions.Build( + "latency", "en", Json("latency", "LATENCY", "Latency"), pool: []); + + Assert.Equal(McOptions.Choices, options.Count); + Assert.Equal("latency", options[correct]); + Assert.Single(options.Where(o => o.Equals("latency", StringComparison.OrdinalIgnoreCase))); + } + + /// + /// A personal pool that is mostly duplicates of one word must not collapse the card. The dedupe + /// is case-insensitive, so "Replica"/"replica" is one distractor, and the hardcoded list has to + /// make up the difference. + /// + [Fact] + public void Build_PoolIsDuplicatesOfOneWord_FillsTheRestAndRepeatsNothing() + { + var pool = new List + { + new("replica", "en"), new("Replica", "en"), new("REPLICA", "en"), + }; + + var (options, correct) = McOptions.Build("latency", "en", distractorsJson: null, pool); + + Assert.Equal(McOptions.Choices, options.Count); + Assert.Equal("latency", options[correct]); + Assert.Equal(options.Count, options.Distinct(StringComparer.OrdinalIgnoreCase).Count()); + } + + /// + /// The answer is never also a filler, even when the hardcoded list contains it verbatim. "river" + /// is in ; saving it as a vocabulary word is ordinary. + /// + [Fact] + public void Build_AnswerIsItselfAHardcodedFiller_IsNotOfferedTwice() + { + var (options, correct) = McOptions.Build("river", "en", distractorsJson: null, pool: []); + + Assert.Equal(McOptions.Choices, options.Count); + Assert.Equal("river", options[correct]); + Assert.Single(options.Where(o => o.Equals("river", StringComparison.OrdinalIgnoreCase))); + } + + /// + /// CorrectIndex must index the list that is returned. Run repeatedly because the order is + /// randomised — a single run can pass by luck on a two-element shuffle. + /// + [Fact] + public void Build_CorrectIndexAlwaysPointsAtTheAnswer_AcrossManyShuffles() + { + for (var i = 0; i < 200; i++) + { + var (options, correct) = McOptions.Build( + "latency", "en", Json("quorum", "replica", "partition", "consensus"), pool: []); + + Assert.InRange(correct, 0, options.Count - 1); + Assert.Equal("latency", options[correct]); + } + } + + /// + /// An empty-string word is not a legal vocabulary row (the save endpoint rejects it), but the + /// builder is now called from three places and should not produce a card whose answer index is + /// -1 if one ever reaches it. + /// + [Fact] + public void Build_BlankWord_DoesNotReturnAnUnfindableCorrectIndex() + { + var (options, correct) = McOptions.Build("", "en", distractorsJson: null, pool: []); + + Assert.InRange(correct, 0, options.Count - 1); + Assert.Equal("", options[correct]); + } +} diff --git a/tests/TextStack.UnitTests/TutorShapeExerciseTests.cs b/tests/TextStack.UnitTests/TutorShapeExerciseTests.cs new file mode 100644 index 000000000..2678001bd --- /dev/null +++ b/tests/TextStack.UnitTests/TutorShapeExerciseTests.cs @@ -0,0 +1,162 @@ +using Api.Endpoints; +using Application.Agents; +using Domain.Entities; + +namespace TextStack.UnitTests; + +/// +/// TutorEndpoints.ShapeExercise — the step #606 added so the calibrated exercise type becomes +/// the card the learner actually gets. Nothing covered it: the PR's own tests are +/// McOptionsTests (the option builder) and TutorEndpointsReplanTests (the re-plan +/// backstops), and neither touches this function. +/// +/// One test here is characterization of a defect, named for it. See its docblock. +/// +public class TutorShapeExerciseTests +{ + private static VocabularyWord Word( + string word = "latency", + string language = "en", + string? translation = null, + string? definition = null, + string? sentence = null, + string? distractors = null) => + new() + { + Id = Guid.NewGuid(), + UserId = Guid.NewGuid(), + SiteId = Guid.NewGuid(), + Word = word, + Language = language, + Translation = translation, + Definition = definition, + Sentence = sentence, + Distractors = distractors, + }; + + [Fact] + public void ShapeExercise_Recall_CarriesNoOptions_SoTheLearnerGradesThemselves() + { + var card = Word(translation: "затримка", sentence: "The latency was unbearable."); + + var (options, correctIndex, blank) = + TutorEndpoints.ShapeExercise(TutorPlanItem.ExerciseRecall, card, [card]); + + Assert.Null(options); + Assert.Null(correctIndex); + Assert.Null(blank); + } + + [Fact] + public void ShapeExercise_Recognition_HasFourOptionsAndNoCloze() + { + var card = Word(translation: "затримка", sentence: "The latency was unbearable."); + + var (options, correctIndex, blank) = + TutorEndpoints.ShapeExercise(TutorPlanItem.ExerciseRecognition, card, [card]); + + Assert.NotNull(options); + Assert.Equal(4, options!.Count); + Assert.Equal("latency", options[correctIndex!.Value]); + // A recognition card is the definition/translation plus four choices — deliberately not the + // sentence, which is what makes `context` a different exercise. + Assert.Null(blank); + } + + [Fact] + public void ShapeExercise_Context_BlanksTheWordOutOfTheSavedSentence() + { + var card = Word(translation: "затримка", sentence: "The latency was unbearable."); + + var (options, _, blank) = + TutorEndpoints.ShapeExercise(TutorPlanItem.ExerciseContext, card, [card]); + + Assert.NotNull(options); + Assert.NotNull(blank); + Assert.DoesNotContain("latency", blank!, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The documented degradation: CalibrateForStage is supposed to make this unreachable, but + /// if a context exercise arrives without a sentence it must become a plain four-option card + /// rather than a cloze with nothing in it. + /// + [Fact] + public void ShapeExercise_ContextWithNoSentence_DegradesToAPlainFourOptionCard() + { + var card = Word(translation: "затримка", sentence: null); + + var (options, correctIndex, blank) = + TutorEndpoints.ShapeExercise(TutorPlanItem.ExerciseContext, card, [card]); + + Assert.NotNull(options); + Assert.Equal(4, options!.Count); + Assert.Equal("latency", options[correctIndex!.Value]); + Assert.Null(blank); + } + + /// + /// The session's other words are the distractor pool, and the card itself is excluded from it — + /// otherwise the answer could be offered twice. + /// + [Fact] + public void ShapeExercise_UsesTheOtherSessionWordsAsDistractors_AndNeverTheCardItself() + { + var card = Word(); + var others = new[] + { + Word("throughput"), Word("backpressure"), Word("jitter"), + }; + var session = new List { card }; + session.AddRange(others); + + var (options, correctIndex, _) = + TutorEndpoints.ShapeExercise(TutorPlanItem.ExerciseRecognition, card, session); + + Assert.NotNull(options); + Assert.Single(options!.Where(o => o.Equals("latency", StringComparison.OrdinalIgnoreCase))); + Assert.Equal("latency", options[correctIndex!.Value]); + // The learner's own words beat the hardcoded filler list. + Assert.All(options.Where((_, i) => i != correctIndex.Value), + o => Assert.Contains(o, new[] { "throughput", "backpressure", "jitter" })); + } + + /// + /// DEFECT (characterized, not fixed). A recognition card is four options and a + /// prompt, and the prompt is built client-side as + /// blankSentence || definition || translation (MultipleChoiceCard.tsx, both + /// clients). recognition has no blankSentence by design. So a vocabulary row saved + /// with neither a translation nor a definition — which the save endpoint permits; only + /// word, language and nativeLanguage are required — reaches the learner as + /// four words and no question at all on web, and as the answer printed above its own + /// options on mobile, whose cascade has a fourth fallback of card.word. + /// + /// Before #606 the Tutor always drew a FlashCard, which shows the word and asks the + /// learner to grade themselves — so it had no prompt to lose. Routing the plan through the MC + /// component is what exposed this. + /// + /// Reproduced live against a running API on 2026-09-12: + /// POST /me/vocabulary/words {"word":"perspicacious","language":"en","nativeLanguage":"uk"} + /// then GET /me/vocabulary/review returns + /// blankSentence:null, definition:null, translation:null, options:[…4…]. + /// + /// When fixed — by refusing to shape an MC exercise with no prompt source and falling back + /// to recall — this test should assert Assert.Null(options). + /// + [Fact] + public void ShapeExercise_RecognitionWithNoTranslationOrDefinition_StillBuildsAPromptlessCard() + { + var card = Word(translation: null, definition: null, sentence: null); + + var (options, _, blank) = + TutorEndpoints.ShapeExercise(TutorPlanItem.ExerciseRecognition, card, [card]); + + Assert.NotNull(options); + Assert.Equal(4, options!.Count); + + // Everything a client could render as the question: + Assert.Null(blank); + Assert.Null(card.Definition); + Assert.Null(card.Translation); + } +}