diff --git a/README.md b/README.md index 88b8cd0..777ce13 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # TypeLearn [![CI](https://github.com/Akay7/TypeLearn/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Akay7/TypeLearn/actions/workflows/ci.yml?query=branch%3Amain) -[![Translation status](https://hosted.weblate.org/widget/typelearn/svg-badge.svg)](https://hosted.weblate.org/engage/typelearn/) [![License](https://img.shields.io/github/license/Akay7/TypeLearn)](LICENSE) [![Top language](https://img.shields.io/github/languages/top/Akay7/TypeLearn)](https://github.com/Akay7/TypeLearn) @@ -37,11 +36,8 @@ Development mode, tests, debugging and running several worktrees are covered in ## Translating -Help translate TypeLearn on [Hosted Weblate](https://hosted.weblate.org/engage/typelearn/). -You don't need to open a pull request: Weblate commits your translations and opens -it for you. Any text not translated yet shows in English. - -To change the English wording or add a language, see +To fix a translation or add a language, open a pull request that edits the JSON files +in `src/frontend/src/locales/`. Any text not translated yet shows in English. See [docs/translation.md](docs/translation.md). ## Audio diff --git a/chart/templates/ingest-job.yaml b/chart/templates/ingest-job.yaml index ff7ec1c..e5fa444 100644 --- a/chart/templates/ingest-job.yaml +++ b/chart/templates/ingest-job.yaml @@ -1,7 +1,22 @@ {{- if .Values.ingest.enabled }} {{- $corpus := .Values.env.CORPUS_PATH | default "/corpus" }} +{{- /* How much to select: a positive integer, or `all`. Checked on its string + form so 100 and "100" are the same value, and refused here rather than in + a Job that starts, runs for a while, and fails. */}} +{{- $count := toString .Values.ingest.count }} +{{- $size := "" }} +{{- if eq $count "all" }} +{{- $size = "--all" }} +{{- else if regexMatch "^[1-9][0-9]*$" $count }} +{{- $size = printf "--count %s" $count }} +{{- else }} +{{- fail (printf "ingest.count must be a positive integer or \"all\", got %q" $count) }} +{{- end }} # Ingestion: read a Common Voice release in place and copy the selected clips -# into the media volume. +# into the media volume, then load the machine translations committed with the +# backend (exercises/fixtures/translations/) for them — so a catalog rebuilt +# from scratch comes back translated without anything being translated again. +# With no fixture yet, the second step says so and does nothing. # # A plain Job, deliberately not a Helm hook. A hook would run on every upgrade, # and this takes minutes and writes into storage every worktree reads — an @@ -40,7 +55,7 @@ spec: command: - sh - -c - - 'python manage.py load_corpus "${CORPUS_PATH:-/corpus}"' + - 'python manage.py load_corpus "${CORPUS_PATH:-/corpus}" {{ $size }} && python manage.py load_translations' envFrom: {{- include "typelearn.envFrom" . | nindent 12 }} env: diff --git a/chart/values-prod.yaml.example b/chart/values-prod.yaml.example index bc775fc..479ea9c 100644 --- a/chart/values-prod.yaml.example +++ b/chart/values-prod.yaml.example @@ -96,6 +96,14 @@ media: accessModes: - ReadWriteMany +# --- Ingestion ---------------------------------------------------------------- +# A deployment loads every sentence the filters admit, not the development +# sample. For the Thai 25.0 release that is 21,429 exercises and as many clips, +# about 400 MB on the media volume, and a Job that runs for minutes. The Job +# still only runs when rendered with ingest.enabled=true; see docs/deployment.md. +ingest: + count: all + # --- Workloads ---------------------------------------------------------------- backend: replicas: 2 diff --git a/chart/values.yaml b/chart/values.yaml index a977bdd..3337da0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -116,6 +116,12 @@ env: # Where ingested clips live in the container; the media volume mounts here. MEDIA_ROOT: /app/media CORPUS_PATH: /corpus + # Every request reaches Django through the Gateway, whose Envoy appends the + # client's address to X-Forwarded-For — so the last entry is the one to + # trust, and the rate limits on ratings and suggestions count by it. Raise it + # by one for each further proxy that appends (a cloud load balancer in front + # of the Gateway); 0 would put every learner behind one shared limit. + TRUSTED_PROXY_COUNT: "1" # Secret values, rendered into a Secret and never into the ConfigMap. Anything # put here is readable by anyone who can read the release, so prefer @@ -209,6 +215,11 @@ worktree: # starts one. ingest: enabled: false + # How much of the corpus to load: a positive integer selects that many of the + # best candidates, `all` selects every sentence the filters admit (about 21k + # for the Thai 25.0 release). Small by default, because a development cluster + # needs a catalog, not the catalog; a deployment asks for `all`. + count: 100 # The release directory as the pod sees it. In development kind mounts the # host's corpus here; set corpusHostPath to mount it from the node. corpusHostPath: "" diff --git a/docs/deployment.md b/docs/deployment.md index a87cf38..ab9fd80 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -96,6 +96,72 @@ one `kubectl delete`, and recovering from a deleted database is not. `externalDatabase` instead, so a managed Postgres is a values change rather than a fork of the chart. +### Loading the exercises + +Ingestion is a Job the chart renders only when asked, never on install or +upgrade: it reads a Common Voice release in place and copies the selected clips +onto the media volume. The corpus has to be on a node, mounted from +`ingest.corpusHostPath`. + +`ingest.count` decides how much of it is loaded. A positive integer selects that +many of the best candidates, and `all` selects every sentence the quality and +beginner filters admit. The chart default is `100`, a development-sized sample; +`values-prod.yaml.example` sets `all`. For the Thai 25.0 release, `all` is 21,429 +exercises and about 400 MB of clips, and the Job runs for minutes, logging its +progress every thousand exercises. Any other value fails at render time. + +```bash +kubectl delete job ingest -n typelearn --ignore-not-found +helm template typelearn ./chart --namespace typelearn --values my-values.yaml \ + --set ingest.enabled=true --set ingest.corpusHostPath=/srv/cv-corpus-25.0-2026-03-09 \ + --show-only templates/ingest-job.yaml | kubectl apply -n typelearn -f - +kubectl logs -n typelearn -f job/ingest +``` + +A Job's spec cannot change once created, which is why the old one is deleted +first. Re-running is safe: exercises already loaded are updated in place, clips +already on the volume are not copied again, and a larger count only adds what +the smaller one left out. Nothing is ever deleted. + +The frontend never downloads the whole catalog, however large it is: each page +load asks for a random deck of 200 (`deck(size: 200)`), and a new deck is drawn +when the learner finishes one. The server caps a deck at +`EXERCISE_DECK_MAX_SIZE` (default 500), settable under `env:` like any other +backend variable. + +### Translations + +A learner can turn on a translation of each phrase into the interface language, +rate it, and suggest a better one. Nothing about this needs setting up for the +application to run: until translations exist, the setting shows "No translation +yet" and offers the suggestion form. + +**Where machine translations come from.** They are committed with the backend, +in `src/backend/exercises/fixtures/translations/th.yaml`, so they ship in the +image. The ingest Job loads them after the corpus, which means a new +deployment, a rebuilt database, or a re-ingest gets them back without +translating anything. A deployment needs no provider key and runs no +translation. The file is made once on a developer's machine (see +[development.md](development.md#4-translations-optional)), and a correction is an +edit to it plus a release. To load a new version without re-ingesting, run +`python manage.py load_translations` in a backend pod. + +**Suggestions and ratings.** Learners are anonymous, so both are identified by a +random id the browser keeps, plus an HMAC of the client address keyed by +`DJANGO_SECRET_KEY` — the address itself is never stored. Both are rate-limited +per hour (`TRANSLATION_PROPOSALS_PER_HOUR`, default 20; `TRANSLATION_RATINGS_PER_HOUR`, +default 300). The address comes from `X-Forwarded-For`, trusting the last +`TRUSTED_PROXY_COUNT` entries; the chart sets 1, for the Gateway. With another +proxy in front of the Gateway that also appends to the header, set it to 2 — too +low, and every learner shares one limit; too high, and a client can choose its own +address. + +A suggestion is never shown until someone approves it. In the admin, under +**Translations**, the list opens on the pending ones, each beside its sentence and +the translation learners see now. **Approve** publishes the selection and +**Reject** hides it for good. Of the published translations of a phrase, learners +see the best rated, and a newly approved one wins a tie. + ### Rendering before installing The chart renders without a cluster, which is what makes it reviewable: diff --git a/docs/development.md b/docs/development.md index 3dd5744..15e0355 100644 --- a/docs/development.md +++ b/docs/development.md @@ -102,11 +102,77 @@ Then press **Ingest the corpus** in the Tilt UI. It selects 100 exercises from clips into the shared media volume. The selection is deterministic — the same corpus always yields the same 100 exercises — and it is safe to re-run. +100 is the chart's default for `ingest.count`, and a deployment sets `all` (see +[deployment.md](deployment.md#loading-the-exercises)). To try the full catalog +locally — about 21k exercises, 400 MB of clips, a few minutes — render the same +Job the button does with the size overridden (only the Job mounts the corpus, so +the backend pod cannot run it). The button reads the backend image from Tilt; +here it is whatever the running backend uses: + +```bash +NS= +IMAGE=$(kubectl get deploy backend -n $NS -o jsonpath='{.spec.template.spec.containers[0].image}') +kubectl delete job ingest -n $NS --ignore-not-found +helm template typelearn chart --namespace $NS --values .tilt/values.yaml \ + --set ingest.enabled=true --set ingest.corpusHostPath=/corpus --set ingest.count=all \ + --set-string image.backend.repository="${IMAGE%:*}" --set-string image.backend.tag="${IMAGE##*:}" \ + --show-only templates/ingest-job.yaml | kubectl apply -n $NS -f - +kubectl logs -n $NS -f job/ingest +``` + +The clips land in the shared `data/media`, so every worktree sees them; the rows +land only in that worktree's database. Running with a count afterwards does not +shrink it again — ingestion adds and updates, it never deletes. + The clips live in `data/media` on the host, shared by every worktree and outliving the cluster, so this only has to happen once even if you delete and recreate the cluster. They are written by the container and owned by a mapped user id; the Tiltfile keeps the directory writable so you can still remove them yourself. +### 4. Translations (optional) + +The **Show translation** setting needs translations to show. They are committed: +`src/backend/exercises/fixtures/translations/th.yaml` holds the machine +translation of every sentence ingestion can select, and **Ingest the corpus** +loads it with `load_translations` after the corpus. Nothing needs a key or a +network. + +The file is made once, on a developer's machine, with the Anthropic API. The key +lives only in that shell. It is not in `.env`, the Secret, or the chart. The +command reads the Common Voice release directly, the same selection ingestion +makes, so it needs neither the cluster nor a database. Start with a trial run: + +```bash +cd src/backend +export ANTHROPIC_API_KEY=sk-ant-... +uv run python manage.py translate_catalog "$TYPELEARN_CORPUS_DIR" --limit 20 +``` + +Read what came back, then run it without `--limit`. For the whole catalog (about +21k sentences into 5 languages) that is about 4,300 requests, a one-time spend. +A re-run only requests the languages an entry is missing, and an interrupted run +keeps everything that was saved: the file is written every 30 seconds and on +exit. `--model` and `--effort` trade quality for cost, and `--languages en,ru` +narrows the targets. Review the diff and commit the file. + +To use a local OpenAI-compatible server instead, such as Lemonade, start it and +pass the model name it lists. No key is needed: + +```bash +uv run python manage.py translate_catalog "$TYPELEARN_CORPUS_DIR" \ + --provider openai-compatible --model --batch-size 10 --limit 20 +``` + +`--base-url` defaults to Lemonade's `http://localhost:8000/api/v1`. Local models +are less reliable than Claude, so keep the batches small and read the trial run. + +To fix a bad translation, edit its line and commit. Loading replaces the text +and clears that translation's ratings, because they were cast on different +words. To try the command out without a key, use `--provider offline --dir +`, which writes placeholders (`[fr] สวัสดี`). It refuses to write +them over the committed file. Suggestions made in the app wait under +**Translations** in the admin (`/admin/`) until they are approved there. + ### Running several worktrees at once Each git worktree gets its own namespace, its own database, its own application diff --git a/docs/translation.md b/docs/translation.md index 06c0f98..44287e4 100644 --- a/docs/translation.md +++ b/docs/translation.md @@ -9,19 +9,16 @@ and any string a language has not translated yet shows in English. Keys are nested objects grouped by component (`settings` → `button` → `label`) and looked up by their dotted path, `t('settings.button.label')`. Don't write a -flat `"settings.button.label"` key: Weblate writes the keys it adds nested, so a -file would end up mixing both shapes. `keys.test.js` fails on dotted keys. +flat `"settings.button.label"` key: a file would end up mixing both shapes. `keys.test.js` fails on dotted keys. -Translations are done on [Hosted Weblate](https://hosted.weblate.org/engage/typelearn/). You don't -need to open a pull request to translate. Weblate commits the changes and opens -the pull request for you. To change the English wording or add a string, edit -`en.json` in a normal pull request. Weblate picks the change up once it merges. +To change a translation, edit that language's file in a normal pull request. To add a +string, add it to `en.json` and to every other language file in the same pull request. A string that depends on a number gets one key per plural form, named the i18next v4 way: `symbolCount_one` and `symbolCount_other` inside `sentence` in `en.json`. Render it with `tPlural(key, count)` from `src/frontend/src/i18n.js`, -not `t`. The helper picks the form the active language needs, so on Weblate each -language gets its own plural forms, such as `_few` and `_many` for Russian. +not `t`. The helper picks the form the active language needs, so each +language has its own plural forms, such as `_few` and `_many` for Russian. ## Adding a language @@ -29,7 +26,7 @@ A new JSON file alone does nothing. The app only loads the languages it lists. T add one, open a pull request that, under `src/frontend/src/`: 1. adds `locales/.json` as an empty `{}`, unless a translator already - started the language on Weblate and its file exists, + started the language and its file exists, 2. in `i18n.js`, imports that file, adds it to the catalogs passed to `createI18n`, and adds the code to `SUPPORTED_LANGUAGES`, 3. adds the language's own name for itself (e.g. "Deutsch") to `LANGUAGES` in @@ -38,5 +35,3 @@ add one, open a pull request that, under `src/frontend/src/`: `components/__tests__/SettingsMenu.test.js`, 4. imports the file in `locales/__tests__/keys.test.js` and adds it to `CATALOGS`, so its keys and placeholders are checked too. - -Once it merges, Weblate lists the language for translators. diff --git a/openspec/changes/add-phrase-translations/.openspec.yaml b/openspec/changes/add-phrase-translations/.openspec.yaml new file mode 100644 index 0000000..0ca5fbe --- /dev/null +++ b/openspec/changes/add-phrase-translations/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-26 diff --git a/openspec/changes/add-phrase-translations/design.md b/openspec/changes/add-phrase-translations/design.md new file mode 100644 index 0000000..19f8b54 --- /dev/null +++ b/openspec/changes/add-phrase-translations/design.md @@ -0,0 +1,346 @@ +## Context + +See proposal.md for the motivation. The current state that shapes this design: + +- The API is read-only and anonymous. `typelearn/urls.py` wraps `/graphql/` in + `csrf_exempt` with a note that it "must be revisited before the first + mutation". This change adds that first mutation. +- There are no user accounts. The session and auth middleware exist only for the + admin. +- The frontend practises from a server-drawn deck of 200 exercises + (`deck(size)`), fetched once and advanced locally. Settings live in + `stores/settings.js` as localStorage-backed refs. The interface language is one + of `SUPPORTED_LANGUAGES` (`en fr de th ru hu`). +- Exercises carry no language field. The corpus is Thai, and the keyboard layout + already defaults on the absence of a language. +- Clips are already treated as a costly, deterministic by-product of the + corpus: they are written once to the media volume (one shared host directory in + development, an RWX PVC in a deployment) and survive database rebuilds. +- A deployment may hold about 21k exercises. Several gunicorn workers and + possibly several pods serve the API, and there is no shared cache service. + +## Goals / Non-Goals + +**Goals:** +- One data model that covers machine translations, learner suggestions, and + ratings, with the "shown" choice cheap to compute for a whole deck. +- Mutations that are safe to expose anonymously: deduplicated, rate-limited, and + moderated before anything a learner wrote is shown to other learners. +- Translation generation that is repeatable, resumable, and paid for once. The + output outlives any one database, and no provider key exists outside the + machine that ran it. + +**Non-Goals:** +- User accounts, or crediting suggestions to a person. +- Translating on demand at request time. +- A moderation UI outside Django admin. +- Putting learner translations or ratings in the fixture. They are runtime + writes, and belong to the database and its backups like any other user data. +- Word-by-word glosses or transliteration. This change covers only + whole-sentence translation. +- Adding a `language` field to `Exercise`. The source language is a setting + until a second corpus language exists. + +## Decisions + +### D1. One `Translation` table for machine and learner text, plus `TranslationRating` + +`Translation(exercise FK, language CharField(max_length=8), text +CharField(max_length=500), origin {machine, learner}, status {published, pending, +rejected}, provider CharField(max_length=64, blank), client_id +CharField(max_length=64, blank), address_hash CharField(max_length=64, blank), +up_votes, down_votes, published_at null, created_at)`. + +`TranslationRating(translation FK, client_id CharField(max_length=64), value +±1, address_hash, updated_at)`, unique on `(translation, client_id)`. + +A suggestion is a `Translation` in `pending`, so approving it is a status flip, +not a copy between tables. A partial unique constraint on `(exercise, language)` +where `origin=machine` makes `load_translations` idempotent as a database +guarantee. Machine rows are a loaded copy of the fixture (D7); learner rows are +original data. A unique constraint on `(exercise, language, text)` stops identical +duplicates, including re-submitting rejected text. + +*Alternative:* separate `Suggestion` and `Translation` tables. Rejected: approval +would duplicate rows, and the admin would need two changelists for the same +kind of thing. + +### D2. Vote counts are denormalized onto `Translation` + +`up_votes` and `down_votes` are updated in the same transaction as the rating +row, with `F()` expressions, using the old and new value of the rating (insert, +change, or delete). The "shown" rule then becomes an ordering: `status=published, +language=L`, ordered by `(up_votes - down_votes) DESC, published_at DESC`. + +For a deck, `Exercise.translation(language)` is resolved with a `Prefetch` of +that ordered queryset. The first element is taken in Python, so 200 exercises +cost one extra query. Strawberry-django's optimizer does not handle a field with +an argument on its own, so the field declares its prefetch explicitly +(`strawberry_django.field(prefetch_related=...)` with a callable that reads the +argument). An index on `(exercise, language, status)` backs it. + +*Alternative:* compute the aggregate from ratings at read time. Rejected: +annotating 200 prefetched rows with an aggregate subquery on every deck fetch +costs more than a counter update on a rare write. + +### D3. Anonymous identity: browser client id plus hashed address + +The frontend generates `crypto.randomUUID()` once and stores it under +`typelearn.clientId`. If storage is unavailable, it keeps the id in memory for +the session. The server validates the client id as a UUID. The client address is +taken from `REMOTE_ADDR`, or from the right-most untrusted `X-Forwarded-For` hop +when `TRUSTED_PROXY_COUNT` > 0. It is stored only as +`HMAC(SECRET_KEY, address)`, never in the clear. + +The client id is trivially spoofable, which is why rate limits also key on the +address hash and why suggestions are moderated. Ratings only reorder +already-published translations, so ballot-stuffing can promote one reviewed text +over another but cannot publish anything new. + +### D4. Rate limits counted from the database + +Limits are checked by counting the caller's own rows in a sliding window: +`Translation(origin=learner)` created per client id and per address hash in the +last hour, and `TranslationRating` updated per client id and per address hash in +the last hour. The limits are settings (`TRANSLATION_PROPOSALS_PER_HOUR`, +default 20; `TRANSLATION_RATINGS_PER_HOUR`, default 300). + +*Alternative:* the Django cache framework. Rejected: the only cache available is +per-process LocMem, which gives each gunicorn worker and pod its own counter. +The rows exist anyway, so counting them is exact everywhere at the cost of one +indexed `COUNT`. + +### D5. Keep the CSRF exemption and replace its comment with the reasoning + +The mutations carry no ambient credential. No cookie or session identifies the +caller; the only identity is `clientId` in the request body. A cross-site forged +request can therefore do nothing an attacker could not do by calling the API +directly. Strawberry's view already refuses to run mutations over GET, and only +JSON bodies are sent by the frontend. The comment in `urls.py` is rewritten to +say that the exemption holds only while no mutation relies on the session. The +schema module docstring is updated to match. + +*Alternative:* drop the exemption and fetch a CSRF token. Rejected: it adds a +round trip and a cookie to protect nothing, and it would need a token endpoint. + +### D6. Mutation errors use stable codes in `extensions` + +Validation failures raise `GraphQLError(message, extensions={'code': ...})` with +codes `NOT_FOUND`, `UNSUPPORTED_LANGUAGE`, `BLANK`, `TOO_LONG`, `DUPLICATE`, +`INVALID_CLIENT_ID`, `NOT_PUBLISHED`, and `RATE_LIMITED`. The frontend maps each +code to an i18n key. It does not match on the English message. + +### D7. Machine translations are a committed YAML fixture, not database rows of record + +The fixture is `src/backend/exercises/fixtures/translations/.yaml`, +committed and shipped in the backend image. `--dir` overrides the directory for +both commands. One entry per source sentence, holding every target language: + +```yaml +format: 1 +source: th +generated_by: anthropic claude-opus-5 +sentences: +- sentence: ฉันชอบกินข้าวผัด + sentence_id: 7f3c2a… + en: I like eating fried rice. + ru: Я люблю есть жареный рис. +``` + +A missing language key means "not translated yet". `generated_by` is +file-level provenance, and it is copied into `Translation.provider`. + +- **Why committed, rather than on a volume:** translation happens once, on a + developer's machine. Committing the result is the only way to get it to every + worktree, CI and every deployment without a Secret, a Job, a volume to seed, + or a copy step, so the provider key never leaves that one shell. The + alternatives each move a key or a file around by hand. Common Voice sentence + text is CC0, so the repository may contain it, even though it deliberately + does not contain the corpus's audio. +- **Why keyed by sentence, not by database id:** database ids differ between + worktrees and rebuilds. `Exercise.sentence` is unique and is exactly what was + translated. `sentence_id` is carried alongside for traceability and as a + fallback match. +- **Why YAML, one entry per sentence:** it can be read, reviewed in a diff, and + corrected by editing one line. All of a sentence's languages sit together. + Unicode is written as itself (`allow_unicode`), keys are written in a fixed + order (sentence, id, then the languages in interface order), and lines never + wrap, so a re-save of the same content is byte-identical. The C loader and + dumper are used when libyaml is present. A full catalog is about 10 MB and + loads in about 2 s. +- **Durability without locks:** one developer runs the command, so there is no + concurrent writer to guard against. Each save writes a temporary file and + `os.replace`s it, so an interruption leaves the previous file whole. Saves + happen at most every 30 seconds and on exit, including Ctrl-C and errors, + which bounds what a crash can lose without rewriting 10 MB after every batch. +- **Corrections:** edit the line. `load_translations` replaces the loaded text + and clears its ratings (D8). + +*Alternative:* a JSONL sidecar per language pair on the media volume, written by +an in-cluster Job with the key in the release Secret (the previous draft). +Rejected: it puts a provider key into every environment that might translate, +needs a Job, an nginx deny rule and file locking on RWX storage, and still needs +a manual copy to seed another deployment. +*Alternative:* keep machine translations only in the database. Rejected: every +fresh database would either pay the provider again or need a dump and restore. + +### D8. `translate_catalog` fills the fixture; `load_translations` imports it + +`translate_catalog CORPUS_ROOT [--locale th] [--manifest validated.tsv] +[--languages en,fr,...] [--provider anthropic|offline] [--model M] [--effort E] +[--no-refusal-fallback] [--batch-size 25] [--limit N] [--dir PATH]` + +- The input is every sentence the ingestion selection admits (`load_corpus`'s + `select_from_release`, with no count), read straight from the release. No + database is touched, so the command runs on a developer's host without the + cluster. Because any ingestion count selects a prefix of that selection, the + fixture covers both a 100-exercise development catalog and a full deployment. +- The source language is `--locale`, defaulting to `TRANSLATION_SOURCE_LANGUAGE` + (`th`). The target default is `SUPPORTED_INTERFACE_LANGUAGES` minus the + source. That list is a backend setting mirroring the frontend's list, because + the backend validates `language` arguments against it too. +- For each target, only the entries missing that language are sent, and new + sentences are appended in selection order. +- Provider options are command flags, not settings. Only this command uses + them, and only on a developer's machine. +- The provider interface is + `translate(sentences: list[str], source: str, target: str) -> list[str]`, + in `exercises/translation/`. `OfflineProvider` returns `"[{target}] {sentence}"` + and requires an explicit `--dir`, so its placeholders cannot land in the + committed file. `AnthropicProvider` sends one batch per request and asks for a + JSON array of exactly `len(sentences)` strings, using structured output. The + array is validated for length, every item must be a non-blank string within + `max_length`, and a batch that fails validation is rejected as a whole. The + key comes from `ANTHROPIC_API_KEY` in the developer's shell. The SDK is a + dev-group dependency and is imported lazily, so the production image has none. + Retries use the SDK's built-in backoff. The default model is `claude-opus-5`, + and `--effort` is the cost lever. A policy decline is re-run on Anthropic's + recommended fallback model in the same call (`fallbacks: "default"`, turned + off with `--no-refusal-fallback`). A refusal that survives that fails only its + batch. An authentication, permission, or unknown-model error stops the whole + run instead, since every batch would fail the same way. +- Failed batches are listed at the end, and the exit code is non-zero if there + were any. The command never touches the database. + +`load_translations [--dir PATH] [--languages ...]` + +- Reads `.yaml`. Each target language present is matched against + `Exercise.sentence` in chunks, falling back to `sentence_id`. +- For each match, the machine row for `(exercise, target)` is created if + missing. If it exists with different text, the text is updated, its + `TranslationRating` rows are deleted, and its counts are zeroed in the same + transaction. If the text is identical, nothing happens. Learner rows are + never touched. A text over `TRANSLATION_MAX_LENGTH` (a hand edit) is skipped + and reported rather than failing the load. +- It reports created, updated, unchanged, and unmatched counts. A missing file + is a quiet no-op. A file that is not a valid fixture fails loudly, since it is + a broken commit. The ingest Job runs it after `load_corpus`. + +*Alternative:* have `translate_catalog` write both fixture and database. +Rejected: two writers of machine rows. One path in, from the file, keeps the +fixture authoritative. + +### D8a. Chart and Tilt + +- The ingest Job's command becomes `load_corpus ... && load_translations`. There + is no translate Job, no provider key in any Secret or `.env`, and nothing in + the Tiltfile. +- The chart sets `TRUSTED_PROXY_COUNT: "1"` for every environment: every + request comes through the Gateway, whose Envoy appends the client's address + to `X-Forwarded-For`. + +### D9. Frontend: its own store, one request per deck and language, separate component + +- `settings.showTranslation` is a persisted boolean, default `false`. The + Settings menu adds a `kind: 'checkbox'` group, reusing the existing checkbox + template. +- The translation state lives in its own `stores/translation.js`, not in the + exercise store: nothing about checking an answer depends on it, and a failure + there must never reach practice. It watches the deck, the setting, and the + interface language. When the setting is on and the interface language is not + the source language, one `exercises(filters: {id: {inList: $ids}}) { + translation(language: $language) }` request fetches whatever the rest of the + deck is missing in that language. The ids go in a variable because 200 of them + inline would exceed `STRAWBERRY_MAX_TOKENS`. Results are cached by + `exerciseId:language`, so switching back to a language costs nothing, and the + deck query is unchanged. The source language is the language of the + exercise's keyboard layout, the same default the keyboard already uses. +- A new `TranslationPanel.vue` sits under the sentence in `SentenceView.vue`. It + shows the text with `lang` set, the thumbs up/down toggles, and a "Suggest a + better translation" disclosure containing a `', 'textarea'))).toBe(true) + expect(typingElsewhere(eventOn('', 'input'))).toBe(true) + }) + + it('is false for the page itself and for buttons', () => { + expect(typingElsewhere({ target: document.body })).toBe(false) + expect(typingElsewhere(eventOn('', 'button'))).toBe(false) + expect(typingElsewhere({ target: null })).toBe(false) + }) +}) diff --git a/src/frontend/src/lib/clientId.js b/src/frontend/src/lib/clientId.js new file mode 100644 index 0000000..a646ce3 --- /dev/null +++ b/src/frontend/src/lib/clientId.js @@ -0,0 +1,30 @@ +const CLIENT_ID_KEY = 'typelearn.clientId' + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + +// Where the id lives when storage refuses to keep it: the rest of this visit. +let sessionId = null + +/** + * This browser's anonymous id, sent with a rating or a suggested translation. + * + * It identifies nothing but the browser, and it proves nothing: it exists so + * that rating a translation twice replaces the first rating rather than adding + * a second, and so the server's rate limits have something to count. Made once + * and kept, so it survives reloads; a browser that refuses storage gets one per + * visit instead. + */ +export function clientId() { + try { + const stored = window.localStorage.getItem(CLIENT_ID_KEY) + if (stored && UUID.test(stored)) { + return stored + } + const fresh = crypto.randomUUID() + window.localStorage.setItem(CLIENT_ID_KEY, fresh) + return fresh + } catch { + sessionId ??= crypto.randomUUID() + return sessionId + } +} diff --git a/src/frontend/src/lib/editable.js b/src/frontend/src/lib/editable.js new file mode 100644 index 0000000..c007b11 --- /dev/null +++ b/src/frontend/src/lib/editable.js @@ -0,0 +1,18 @@ +/** + * Whether a key event belongs to a text field other than the answer. + * + * The exercise listens for some keys on the whole document — Enter to move on + * from the summary — because the answer field is not always the one focused. + * But a learner writing in another field (a suggested translation) is typing + * prose, not answering, and none of those keys mean anything there. The answer + * field is marked `data-answer-field`; every other editable element is + * "elsewhere". + */ +export function typingElsewhere(event) { + const target = event.target + if (!target || typeof target.closest !== 'function') return false + if (target.closest('[data-answer-field]')) return false + return Boolean( + target.isContentEditable || target.closest('textarea, input, select, [contenteditable="true"]'), + ) +} diff --git a/src/frontend/src/lib/graphql.js b/src/frontend/src/lib/graphql.js new file mode 100644 index 0000000..04588fe --- /dev/null +++ b/src/frontend/src/lib/graphql.js @@ -0,0 +1,48 @@ +// A path, never a URL. The application is served from one origin in every +// environment — behind the Gateway in the cluster and in a deployment, and +// behind the dev server's proxy when the frontend runs on the host — so the +// backend is always reachable at a path on the current origin. An absolute URL +// here would be a second origin, and the backend carries no CORS configuration +// to make one work. +export const API_URL = import.meta.env.VITE_API_URL ?? '/graphql/' + +/** + * A request the API refused. `code` is the machine-readable reason the schema + * attaches to a refusal (`RATE_LIMITED`, `BLANK`, …), or null for a failure + * that carries none — a server error, a malformed query. + */ +export class GraphQLRequestError extends Error { + constructor(message, code = null) { + super(message) + this.name = 'GraphQLRequestError' + this.code = code + } +} + +/** + * POST one document and return its `data`. A refusal from the API or a non-2xx + * answer throws a GraphQLRequestError; a request that never got an answer + * throws whatever `fetch` threw, untouched. + */ +export async function graphql(query, variables = {}) { + const response = await fetch(API_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables }), + }) + + if (!response.ok) { + throw new GraphQLRequestError(`The backend responded ${response.status}`) + } + + const body = await response.json() + + // A GraphQL error arrives as HTTP 200 with an `errors` array, so checking + // response.ok alone would report success on a broken query. + if (body.errors?.length) { + const [first] = body.errors + throw new GraphQLRequestError(first.message, first.extensions?.code ?? null) + } + + return body.data +} diff --git a/src/frontend/src/locales/__tests__/keys.test.js b/src/frontend/src/locales/__tests__/keys.test.js index deaa0b0..05a9613 100644 --- a/src/frontend/src/locales/__tests__/keys.test.js +++ b/src/frontend/src/locales/__tests__/keys.test.js @@ -23,8 +23,8 @@ function flatten(catalog, prefix = '') { const CATALOGS = { en: enCatalog, fr: frCatalog, de: deCatalog, th: thCatalog, ru: ruCatalog, hu: huCatalog } // en.json is the reference catalog (see the comment in i18n.js). The other -// locales are edited on Weblate, and a language is often only partly translated -// there, so a key that is missing or blank passes: it falls back to English. +// locales are edited by volunteers, and a language is often only partly translated, +// so a key that is missing or blank passes: it falls back to English. // What does not pass is a translation that would break at runtime — a key en.json // no longer has, or a placeholder that does not match the English one. const en = flatten(enCatalog) @@ -67,8 +67,7 @@ describe('locale catalogs', () => { } }) - // A flat 'a.b' key would still resolve, but Weblate writes the keys it adds - // nested — so a catalog that mixed both shapes would not stay consistent. + // A flat 'a.b' key would still resolve, but a catalog that mixed both shapes would not stay consistent. for (const [locale, catalog] of Object.entries(CATALOGS)) { it(`${locale}.json nests its keys instead of joining them with dots`, () => { expect(dottedKeys(catalog)).toEqual([]) diff --git a/src/frontend/src/locales/de.json b/src/frontend/src/locales/de.json index 2b0efd4..5b8bc14 100644 --- a/src/frontend/src/locales/de.json +++ b/src/frontend/src/locales/de.json @@ -79,5 +79,27 @@ }, "footer": { "source": "Quellcode auf GitHub" + }, + "translation": { + "loading": "Übersetzung wird geladen…", + "unavailable": "Die Übersetzung konnte nicht geladen werden.", + "none": "Noch keine Übersetzung.", + "machine": "Maschinelle Übersetzung", + "expand": "Ganze Übersetzung anzeigen", + "rateGood": "Gute Übersetzung", + "rateBad": "Schlechte Übersetzung", + "suggest": "Übersetzung vorschlagen", + "suggestBetter": "Bessere Übersetzung vorschlagen", + "placeholder": "Deine Übersetzung", + "submit": "Senden", + "cancel": "Abbrechen", + "thanks": "Danke – sie erscheint nach der Prüfung.", + "errors": { + "BLANK": "Schreibe zuerst eine Übersetzung.", + "TOO_LONG": "Diese Übersetzung ist zu lang.", + "DUPLICATE": "Diese Übersetzung wurde schon vorgeschlagen.", + "RATE_LIMITED": "Zu viele auf einmal. Versuche es später erneut.", + "generic": "Das hat nicht geklappt. Versuche es erneut." + } } } diff --git a/src/frontend/src/locales/en.json b/src/frontend/src/locales/en.json index 2094480..79c6f21 100644 --- a/src/frontend/src/locales/en.json +++ b/src/frontend/src/locales/en.json @@ -35,6 +35,10 @@ }, "language": { "title": "Language" + }, + "translation": { + "label": "Show translation", + "description": "The phrase in your language, beneath it" } }, "sentence": { @@ -79,5 +83,27 @@ }, "footer": { "source": "Source code on GitHub" + }, + "translation": { + "loading": "Loading the translation…", + "unavailable": "The translation could not be loaded.", + "none": "No translation yet.", + "machine": "Machine translation", + "expand": "Show the whole translation", + "rateGood": "Good translation", + "rateBad": "Bad translation", + "suggest": "Suggest a translation", + "suggestBetter": "Suggest a better translation", + "placeholder": "Your translation", + "submit": "Send", + "cancel": "Cancel", + "thanks": "Thanks — it will appear after review.", + "errors": { + "BLANK": "Write a translation first.", + "TOO_LONG": "That translation is too long.", + "DUPLICATE": "That translation has already been suggested.", + "RATE_LIMITED": "Too many at once. Try again later.", + "generic": "That did not go through. Try again." + } } } diff --git a/src/frontend/src/locales/fr.json b/src/frontend/src/locales/fr.json index 3fc8f92..6efcbbb 100644 --- a/src/frontend/src/locales/fr.json +++ b/src/frontend/src/locales/fr.json @@ -80,5 +80,27 @@ }, "footer": { "source": "Code source sur GitHub" + }, + "translation": { + "loading": "Chargement de la traduction…", + "unavailable": "La traduction n’a pas pu être chargée.", + "none": "Pas encore de traduction.", + "machine": "Traduction automatique", + "expand": "Afficher toute la traduction", + "rateGood": "Bonne traduction", + "rateBad": "Mauvaise traduction", + "suggest": "Proposer une traduction", + "suggestBetter": "Proposer une meilleure traduction", + "placeholder": "Votre traduction", + "submit": "Envoyer", + "cancel": "Annuler", + "thanks": "Merci — elle apparaîtra après relecture.", + "errors": { + "BLANK": "Écrivez d’abord une traduction.", + "TOO_LONG": "Cette traduction est trop longue.", + "DUPLICATE": "Cette traduction a déjà été proposée.", + "RATE_LIMITED": "Trop de demandes à la fois. Réessayez plus tard.", + "generic": "Cela n’a pas fonctionné. Réessayez." + } } } diff --git a/src/frontend/src/locales/hu.json b/src/frontend/src/locales/hu.json index 57349c0..8465f0a 100644 --- a/src/frontend/src/locales/hu.json +++ b/src/frontend/src/locales/hu.json @@ -79,5 +79,27 @@ }, "footer": { "source": "Forráskód a GitHubon" + }, + "translation": { + "loading": "Fordítás betöltése…", + "unavailable": "A fordítást nem sikerült betölteni.", + "none": "Még nincs fordítás.", + "machine": "Gépi fordítás", + "expand": "A teljes fordítás megjelenítése", + "rateGood": "Jó fordítás", + "rateBad": "Rossz fordítás", + "suggest": "Fordítás javaslata", + "suggestBetter": "Jobb fordítás javaslata", + "placeholder": "Az ön fordítása", + "submit": "Küldés", + "cancel": "Mégse", + "thanks": "Köszönjük — felülvizsgálat után jelenik meg.", + "errors": { + "BLANK": "Először írjon egy fordítást.", + "TOO_LONG": "Ez a fordítás túl hosszú.", + "DUPLICATE": "Ezt a fordítást már javasolták.", + "RATE_LIMITED": "Túl sok kérés egyszerre. Próbálja újra később.", + "generic": "Nem sikerült. Próbálja újra." + } } } diff --git a/src/frontend/src/locales/ru.json b/src/frontend/src/locales/ru.json index 4104958..12d0e79 100644 --- a/src/frontend/src/locales/ru.json +++ b/src/frontend/src/locales/ru.json @@ -80,5 +80,27 @@ }, "footer": { "source": "Исходный код на GitHub" + }, + "translation": { + "loading": "Загрузка перевода…", + "unavailable": "Не удалось загрузить перевод.", + "none": "Перевода пока нет.", + "machine": "Машинный перевод", + "expand": "Показать весь перевод", + "rateGood": "Хороший перевод", + "rateBad": "Плохой перевод", + "suggest": "Предложить перевод", + "suggestBetter": "Предложить лучший перевод", + "placeholder": "Ваш перевод", + "submit": "Отправить", + "cancel": "Отмена", + "thanks": "Спасибо — перевод появится после проверки.", + "errors": { + "BLANK": "Сначала напишите перевод.", + "TOO_LONG": "Этот перевод слишком длинный.", + "DUPLICATE": "Такой перевод уже предложен.", + "RATE_LIMITED": "Слишком много запросов. Попробуйте позже.", + "generic": "Не получилось. Попробуйте ещё раз." + } } } diff --git a/src/frontend/src/locales/th.json b/src/frontend/src/locales/th.json index 5f94af9..8fb7391 100644 --- a/src/frontend/src/locales/th.json +++ b/src/frontend/src/locales/th.json @@ -78,5 +78,27 @@ }, "footer": { "source": "ซอร์สโค้ดบน GitHub" + }, + "translation": { + "loading": "กำลังโหลดคำแปล…", + "unavailable": "ไม่สามารถโหลดคำแปลได้", + "none": "ยังไม่มีคำแปล", + "machine": "คำแปลโดยเครื่อง", + "expand": "แสดงคำแปลทั้งหมด", + "rateGood": "คำแปลดี", + "rateBad": "คำแปลไม่ดี", + "suggest": "เสนอคำแปล", + "suggestBetter": "เสนอคำแปลที่ดีกว่า", + "placeholder": "คำแปลของคุณ", + "submit": "ส่ง", + "cancel": "ยกเลิก", + "thanks": "ขอบคุณ — คำแปลจะปรากฏหลังตรวจสอบ", + "errors": { + "BLANK": "โปรดเขียนคำแปลก่อน", + "TOO_LONG": "คำแปลนี้ยาวเกินไป", + "DUPLICATE": "มีการเสนอคำแปลนี้แล้ว", + "RATE_LIMITED": "คำขอมากเกินไป โปรดลองใหม่ภายหลัง", + "generic": "ไม่สำเร็จ โปรดลองอีกครั้ง" + } } } diff --git a/src/frontend/src/stores/__tests__/exercise.test.js b/src/frontend/src/stores/__tests__/exercise.test.js index 82c142b..c5bfde8 100644 --- a/src/frontend/src/stores/__tests__/exercise.test.js +++ b/src/frontend/src/stores/__tests__/exercise.test.js @@ -307,15 +307,18 @@ describe('the sentence the learner is asked for', () => { }) }) +/** A fetch response answering the Deck query with `deck`. */ +function deckResponse(deck) { + return { ok: true, json: async () => ({ data: { deck } }) } +} + +function exercisesFor(ids) { + return ids.map((id) => ({ id, sentence: 'สวัสดี', audioUrl: `/${id}.mp3`, difficulty: 1 })) +} + describe('loading the catalog', () => { - function respondWith(exercises) { - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ data: { exercises } }), - }), - ) + function respondWith(deck) { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(deckResponse(deck))) } afterEach(() => { @@ -359,3 +362,92 @@ describe('loading the catalog', () => { expect(store.current).toBeNull() }) }) + +describe('the deck comes from the server', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('asks for a deck of a fixed size, never the whole catalog', async () => { + const fetch = vi.fn().mockResolvedValue(deckResponse(exercisesFor(['1']))) + vi.stubGlobal('fetch', fetch) + + await useExerciseStore().load() + + const { query } = JSON.parse(fetch.mock.calls[0][1].body) + expect(query).toMatch(/deck\(size: 200\)/) + expect(query).not.toMatch(/exercises/) + }) + + it('keeps the order the server drew', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(deckResponse(exercisesFor(['3', '1', '2'])))) + + const store = useExerciseStore() + await store.load() + + expect(store.deck.map((exercise) => exercise.id)).toEqual(['3', '1', '2']) + }) + + it('draws a new deck once the last exercise is done, and starts it', async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(deckResponse(exercisesFor(['a1', 'a2']))) + .mockResolvedValueOnce(deckResponse(exercisesFor(['b1', 'b2']))) + vi.stubGlobal('fetch', fetch) + + const store = useExerciseStore() + await store.load() + store.next() + expect(store.current.id).toBe('a2') + + const refill = store.next() + // Still the answered exercise, and never a loading state, while the draw is out. + expect(store.current.id).toBe('a2') + expect(store.status).toBe('ready') + await refill + + expect(fetch).toHaveBeenCalledTimes(2) + expect(store.current.id).toBe('b1') + expect(store.status).toBe('ready') + expect(store.typed).toBe('') + expect(store.result).toBe(null) + }) + + it('asks only once when the learner advances again while a draw is out', async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(deckResponse(exercisesFor(['a1']))) + .mockResolvedValue(deckResponse(exercisesFor(['b1']))) + vi.stubGlobal('fetch', fetch) + + const store = useExerciseStore() + await store.load() + const refill = store.next() + store.next() + await refill + + expect(fetch).toHaveBeenCalledTimes(2) + }) + + it('starts the deck it has again when a new one cannot be drawn', async () => { + const failure = new Error('network down') + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce(deckResponse(exercisesFor(['a1', 'a2']))) + .mockRejectedValueOnce(failure), + ) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + + const store = useExerciseStore() + await store.load() + store.next() + await store.next() + + expect(store.current.id).toBe('a1') + expect(store.status).toBe('ready') + expect(consoleError).toHaveBeenCalledWith(expect.any(String), failure) + consoleError.mockRestore() + }) +}) diff --git a/src/frontend/src/stores/__tests__/settings.test.js b/src/frontend/src/stores/__tests__/settings.test.js index 281c747..8cc9fd1 100644 --- a/src/frontend/src/stores/__tests__/settings.test.js +++ b/src/frontend/src/stores/__tests__/settings.test.js @@ -191,6 +191,38 @@ describe('the completion-stats setting', () => { }) }) +describe('the show-translation setting', () => { + it('defaults to hidden', async () => { + const store = await freshStore() + expect(store.showTranslation).toBe(false) + }) + + it('is read back by a fresh store instance', async () => { + const first = await freshStore() + first.showTranslation = true + + const second = await freshStore() + expect(second.showTranslation).toBe(true) + }) + + it('does not crash store creation when storage throws on read', async () => { + window.localStorage.broken() + + const store = await freshStore() + expect(store.showTranslation).toBe(false) + }) + + it('still applies for the session when storage throws on write', async () => { + const store = await freshStore() + window.localStorage.broken() + + expect(() => { + store.showTranslation = true + }).not.toThrow() + expect(store.showTranslation).toBe(true) + }) +}) + describe('the interface language', () => { it("defaults to the browser's language when it is supported", async () => { vi.stubGlobal('navigator', { language: 'fr-CA' }) diff --git a/src/frontend/src/stores/__tests__/translation.test.js b/src/frontend/src/stores/__tests__/translation.test.js new file mode 100644 index 0000000..bf8aee4 --- /dev/null +++ b/src/frontend/src/stores/__tests__/translation.test.js @@ -0,0 +1,286 @@ +import { createPinia, setActivePinia } from 'pinia' +import { nextTick } from 'vue' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { useExerciseStore } from '../exercise' +import { useSettingsStore } from '../settings' +import { errorKey, useTranslationStore } from '../translation' + +function fakeStorage() { + const data = new Map() + return { + getItem: (key) => (data.has(key) ? data.get(key) : null), + setItem: (key, value) => data.set(key, String(value)), + } +} + +const translationOf = (id, language, extra = {}) => ({ + id: `t${id}-${language}`, + text: `[${language}] sentence ${id}`, + origin: 'MACHINE', + upVotes: 0, + downVotes: 0, + ...extra, +}) + +/** + * A fetch that answers the translations query from `known` (exercise id -> + * language -> translation) and hands every mutation to `onMutation`. + */ +function backend({ known = {}, onMutation = () => ({}) } = {}) { + return vi.fn(async (url, init) => { + const { query, variables } = JSON.parse(init.body) + let body + if (query.includes('query Translations')) { + body = { + data: { + exercises: variables.ids.map((id) => ({ + id, + translation: known[id]?.[variables.language] ?? null, + })), + }, + } + } else { + body = onMutation(query, variables) + } + return { ok: true, status: 200, json: async () => body } + }) +} + +const requestsOf = (fetch, name) => + fetch.mock.calls + .map(([, init]) => JSON.parse(init.body)) + .filter(({ query }) => query.includes(name)) + +/** Let every pending promise and watcher run. */ +async function settle() { + for (let i = 0; i < 5; i += 1) { + await nextTick() + await Promise.resolve() + } +} + +function setup({ deck = ['1', '2', '3'], show = true, language = 'en' } = {}) { + const settings = useSettingsStore() + settings.showTranslation = show + settings.interfaceLanguage = language + const exercise = useExerciseStore() + exercise.deck = deck.map((id) => ({ id, sentence: 'สวัสดี', audioUrl: `/${id}.mp3` })) + exercise.index = 0 + exercise.status = 'ready' + return { settings, exercise, translation: useTranslationStore() } +} + +beforeEach(() => { + setActivePinia(createPinia()) + globalThis.window = globalThis.window ?? {} + window.localStorage = fakeStorage() + vi.stubGlobal('navigator', { language: 'en-US' }) + vi.spyOn(console, 'error').mockImplementation(() => {}) +}) + +afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() +}) + +describe('fetching translations', () => { + it('asks for nothing while the setting is off', async () => { + const fetch = backend() + vi.stubGlobal('fetch', fetch) + + setup({ show: false }) + await settle() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('fetches the rest of the deck in one request once turned on', async () => { + const fetch = backend({ known: { 1: { en: translationOf(1, 'en') } } }) + vi.stubGlobal('fetch', fetch) + const { settings, translation } = setup({ show: false }) + + settings.showTranslation = true + await settle() + + const [request] = requestsOf(fetch, 'query Translations') + expect(request.variables).toEqual({ ids: ['1', '2', '3'], language: 'en' }) + expect(translation.state).toBe('ready') + expect(translation.current.text).toBe('[en] sentence 1') + }) + + it('reports an exercise with no translation as none', async () => { + vi.stubGlobal('fetch', backend()) + const { translation } = setup() + await settle() + + expect(translation.state).toBe('none') + expect(translation.current).toBe(null) + }) + + it('fetches again in the new language when the interface language changes', async () => { + const fetch = backend({ + known: { 1: { en: translationOf(1, 'en'), fr: translationOf(1, 'fr') } }, + }) + vi.stubGlobal('fetch', fetch) + const { settings, translation } = setup() + await settle() + + settings.interfaceLanguage = 'fr' + await settle() + + expect(requestsOf(fetch, 'query Translations').map((r) => r.variables.language)).toEqual([ + 'en', + 'fr', + ]) + expect(translation.current.text).toBe('[fr] sentence 1') + + // Back to English: already cached, so no third request. + settings.interfaceLanguage = 'en' + await settle() + expect(requestsOf(fetch, 'query Translations')).toHaveLength(2) + expect(translation.current.text).toBe('[en] sentence 1') + }) + + it('never asks for a translation into the exercise’s own language', async () => { + const fetch = backend() + vi.stubGlobal('fetch', fetch) + + const { translation } = setup({ language: 'th' }) + await settle() + + expect(translation.active).toBe(false) + expect(fetch).not.toHaveBeenCalled() + }) + + it('reports a failed request without touching practice', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network down'))) + const { exercise, translation } = setup() + await settle() + + expect(translation.state).toBe('error') + expect(exercise.status).toBe('ready') + }) + + it('only asks for exercises not yet fetched when a new deck arrives', async () => { + const fetch = backend() + vi.stubGlobal('fetch', fetch) + const { exercise } = setup({ deck: ['1', '2'] }) + await settle() + + exercise.deck = [...exercise.deck, { id: '9', sentence: 'ก', audioUrl: '/9.mp3' }] + await settle() + + expect(requestsOf(fetch, 'query Translations').map((r) => r.variables.ids)).toEqual([ + ['1', '2'], + ['9'], + ]) + }) +}) + +describe('rating', () => { + function rated(value) { + return { data: { rateTranslation: translationOf(1, 'en', value) } } + } + + it('sends the rating and shows the counts the server returns', async () => { + const fetch = backend({ + known: { 1: { en: translationOf(1, 'en') } }, + onMutation: () => rated({ upVotes: 1 }), + }) + vi.stubGlobal('fetch', fetch) + const { translation } = setup() + await settle() + + expect(await translation.rate('UP')).toBe(null) + + const [request] = requestsOf(fetch, 'mutation Rate') + expect(request.variables.value).toBe('UP') + expect(request.variables.clientId).toMatch(/^[0-9a-f-]{36}$/) + expect(translation.currentRating).toBe('UP') + expect(translation.current.upVotes).toBe(1) + }) + + it('withdraws a rating chosen twice', async () => { + const fetch = backend({ + known: { 1: { en: translationOf(1, 'en') } }, + onMutation: () => rated({}), + }) + vi.stubGlobal('fetch', fetch) + const { translation } = setup() + await settle() + + await translation.rate('DOWN') + await translation.rate('DOWN') + + expect(requestsOf(fetch, 'mutation Rate').map((r) => r.variables.value)).toEqual([ + 'DOWN', + 'NONE', + ]) + expect(translation.currentRating).toBe(undefined) + }) + + it('names the refusal when rating is rate-limited', async () => { + vi.stubGlobal( + 'fetch', + backend({ + known: { 1: { en: translationOf(1, 'en') } }, + onMutation: () => ({ errors: [{ message: 'no', extensions: { code: 'RATE_LIMITED' } }] }), + }), + ) + const { translation } = setup() + await settle() + + expect(await translation.rate('UP')).toBe('translation.errors.RATE_LIMITED') + expect(translation.currentRating).toBe(undefined) + }) +}) + +describe('proposing', () => { + it('sends a trimmed suggestion for the current exercise and language', async () => { + const fetch = backend({ onMutation: () => ({ data: { proposeTranslation: { accepted: true } } }) }) + vi.stubGlobal('fetch', fetch) + const { translation } = setup({ language: 'ru' }) + await settle() + + expect(await translation.propose(' Привет ')).toBe(null) + + const [request] = requestsOf(fetch, 'mutation Propose') + expect(request.variables).toMatchObject({ exerciseId: '1', language: 'ru', text: 'Привет' }) + }) + + it('refuses the obvious without a request', async () => { + const fetch = backend({ known: { 1: { en: translationOf(1, 'en') } } }) + vi.stubGlobal('fetch', fetch) + const { translation } = setup() + await settle() + + expect(await translation.propose(' ')).toBe('translation.errors.BLANK') + expect(await translation.propose('x'.repeat(501))).toBe('translation.errors.TOO_LONG') + expect(await translation.propose('[en] sentence 1')).toBe('translation.errors.DUPLICATE') + expect(requestsOf(fetch, 'mutation Propose')).toHaveLength(0) + }) + + it('maps server refusals to messages, and unknown ones to the generic one', async () => { + const code = { value: 'DUPLICATE' } + vi.stubGlobal( + 'fetch', + backend({ + onMutation: () => ({ errors: [{ message: 'no', extensions: { code: code.value } }] }), + }), + ) + const { translation } = setup() + await settle() + + expect(await translation.propose('Hello')).toBe('translation.errors.DUPLICATE') + code.value = 'SOMETHING_NEW' + expect(await translation.propose('Hello')).toBe('translation.errors.generic') + }) +}) + +describe('errorKey', () => { + it('falls back to the generic message for errors with no code', () => { + expect(errorKey(new Error('network down'))).toBe('translation.errors.generic') + expect(errorKey({ code: 'TOO_LONG' })).toBe('translation.errors.TOO_LONG') + }) +}) diff --git a/src/frontend/src/stores/exercise.js b/src/frontend/src/stores/exercise.js index 885934a..d59f394 100644 --- a/src/frontend/src/stores/exercise.js +++ b/src/frontend/src/stores/exercise.js @@ -2,24 +2,20 @@ import { defineStore } from 'pinia' import { computed, ref, watch } from 'vue' import { compare, dropLast, isComplete, nextExpected } from '../lib/checking' +import { graphql } from '../lib/graphql' import { layoutFor, layoutsAvailable, withoutUnreachable } from '../lib/keyboard' import { useSettingsStore } from './settings' import { useStatsStore } from './stats' -// A path, never a URL. The application is served from one origin in every -// environment — behind the Gateway in the cluster and in a deployment, and -// behind the dev server's proxy when the frontend runs on the host — so the -// backend is always reachable at a path on the current origin. An absolute URL -// here would be a second origin, and the backend carries no CORS configuration -// to make one work. -const API_URL = import.meta.env.VITE_API_URL ?? '/graphql/' - -// The whole catalog, in one request. It is 100 rows, and fetching it up front -// is what makes advancing to the next exercise cost nothing: the moment after a -// green result is the worst possible time to show a loading state. -const CATALOG_QUERY = ` - query Catalog { - exercises { +// A random sample of the catalog, never the whole of it: a deployment holds +// tens of thousands of exercises. Fetched up front, so advancing within a deck +// costs nothing — the moment after a green result is the worst possible time +// to show a loading state. The server shuffles, so the client does not. +const DECK_SIZE = 200 + +const DECK_QUERY = ` + query Deck { + deck(size: ${DECK_SIZE}) { id sentence audioUrl @@ -31,21 +27,30 @@ const CATALOG_QUERY = ` // How long the verdict stays on screen before the next exercise replaces it. const ADVANCE_DELAY_MS = 900 -/** Fisher-Yates, in place: every ordering equally likely, no sort comparator abuse. */ -function shuffle(items) { - for (let i = items.length - 1; i > 0; i -= 1) { - const j = Math.floor(Math.random() * (i + 1)) - ;[items[i], items[j]] = [items[j], items[i]] - } - return items +/** + * Fetch a deck, keeping only the exercises some keyboard can type. + * + * An exercise whose sentence holds nothing any keyboard can type is not an + * exercise: presented, it would be complete before the learner typed anything. + * Dropped here rather than hidden later, so the deck is the list of things + * there are to practise. + */ +async function fetchDeck() { + const data = await graphql(DECK_QUERY) + + return data.deck.filter( + (exercise) => + withoutUnreachable(exercise.sentence, layoutsAvailable(layoutFor(exercise.language))) !== + '', + ) } export const useExerciseStore = defineStore('exercise', () => { const settingsStore = useSettingsStore() const statsStore = useStatsStore() - // The catalog in the order this session will practise it, and where we are in - // it. Shuffled once at load, so no exercise repeats until it wraps. + // The deck being practised, in the order the server drew it, and where we + // are in it. No exercise repeats within a deck; the next deck is a fresh draw. const deck = ref([]) const index = ref(0) // 'loading' | 'ready' | 'error' | 'empty' — what the view renders. @@ -128,37 +133,7 @@ export const useExerciseStore = defineStore('exercise', () => { status.value = 'loading' try { - const response = await fetch(API_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query: CATALOG_QUERY }), - }) - - if (!response.ok) { - throw new Error(`The backend responded ${response.status}`) - } - - const body = await response.json() - - // A GraphQL error arrives as HTTP 200 with an `errors` array, so checking - // response.ok alone would report success on a broken query. - if (body.errors?.length) { - throw new Error(body.errors[0].message) - } - - // An exercise whose sentence holds nothing any keyboard can type is not - // an exercise: presented, it would be complete before the learner typed - // anything. Dropped here rather than hidden later, so the deck is the - // list of things there are to practise. - const practisable = body.data.exercises.filter( - (exercise) => - withoutUnreachable( - exercise.sentence, - layoutsAvailable(layoutFor(exercise.language)), - ) !== '', - ) - - deck.value = shuffle(practisable) + deck.value = await fetchDeck() index.value = 0 resetTyped() result.value = null @@ -279,8 +254,47 @@ export const useExerciseStore = defineStore('exercise', () => { } } + function present(position) { + index.value = position + resetTyped() + result.value = null + } + + // Held while a new deck is on its way, so a second advance in the meantime + // does not ask for another. + let refilling = false + /** - * Present the next exercise, wrapping to the start when the deck runs out. + * Replace a finished deck with a fresh draw, in the background. + * + * `status` stays 'ready': the answered exercise and its verdict stay on + * screen until the new deck arrives, which is usually within the pause the + * verdict is shown for anyway. If the draw fails, or comes back with nothing + * to practise, the deck already held starts again rather than practice + * ending in an error. + */ + async function refill() { + if (refilling) { + return + } + refilling = true + + try { + const fresh = await fetchDeck() + if (fresh.length) { + deck.value = fresh + } + } catch (error) { + console.error('Could not load a new deck:', error) + } finally { + refilling = false + } + + present(0) + } + + /** + * Present the next exercise, drawing a new deck when this one runs out. * Everything belonging to the previous exercise is cleared here, so no * component has to remember to reset itself. */ @@ -291,9 +305,12 @@ export const useExerciseStore = defineStore('exercise', () => { return } - index.value = (index.value + 1) % deck.value.length - resetTyped() - result.value = null + if (index.value + 1 < deck.value.length) { + present(index.value + 1) + return + } + + return refill() } return { diff --git a/src/frontend/src/stores/settings.js b/src/frontend/src/stores/settings.js index ae04aa4..60f3634 100644 --- a/src/frontend/src/stores/settings.js +++ b/src/frontend/src/stores/settings.js @@ -8,6 +8,7 @@ const OVERRIDE_KEY = 'typelearn.virtualKeyboardOverride' const KEYBOARD_KEY = 'typelearn.onScreenKeyboardVisible' const COMPLETION_STATS_KEY = 'typelearn.showCompletionStats' const LANGUAGE_KEY = 'typelearn.interfaceLanguage' +const SHOW_TRANSLATION_KEY = 'typelearn.showTranslation' // 'auto' follows the device classification; 'on'/'off' force the virtual // keyboard either way, for the cases classification gets wrong in either @@ -75,6 +76,25 @@ function saveCompletionStatsVisible(value) { } } +/** Hidden by default, unlike the two above: working out what a sentence means + * from its sound and script is part of the practice, so the translation is + * something a learner asks for rather than something they have to dismiss. */ +function loadShowTranslation() { + try { + return window.localStorage.getItem(SHOW_TRANSLATION_KEY) === 'true' + } catch { + return false + } +} + +function saveShowTranslation(value) { + try { + window.localStorage.setItem(SHOW_TRANSLATION_KEY, String(value)) + } catch { + // Storage unavailable — the choice still applies for this session. + } +} + /** The base subtag of a BCP-47 tag ('fr-CA' -> 'fr'), lowercased — none of * the six catalogs are regional variants, so a region suffix is dropped * rather than treated as a mismatch. */ @@ -127,6 +147,8 @@ export const useSettingsStore = defineStore('settings', () => { // has (verdict, then straight to the next exercise). const showCompletionStats = ref(loadCompletionStatsVisible()) const interfaceLanguage = ref(loadLanguage()) + // Whether the exercise shows its translation into the interface language. + const showTranslation = ref(loadShowTranslation()) // Synchronous, not batched: the choice should survive a tab closed right // after it's made, not wait for a tick that might not come. @@ -134,6 +156,7 @@ export const useSettingsStore = defineStore('settings', () => { watch(onScreenKeyboardVisible, saveKeyboardVisible, { flush: 'sync' }) watch(showCompletionStats, saveCompletionStatsVisible, { flush: 'sync' }) watch(interfaceLanguage, saveLanguage, { flush: 'sync' }) + watch(showTranslation, saveShowTranslation, { flush: 'sync' }) // The one place `interfaceLanguage` drives what the rest of the app reads: // every component keeps reading this store for every setting, rather than @@ -162,5 +185,6 @@ export const useSettingsStore = defineStore('settings', () => { onScreenKeyboardVisible, showCompletionStats, interfaceLanguage, + showTranslation, } }) diff --git a/src/frontend/src/stores/translation.js b/src/frontend/src/stores/translation.js new file mode 100644 index 0000000..8088ade --- /dev/null +++ b/src/frontend/src/stores/translation.js @@ -0,0 +1,201 @@ +import { defineStore } from 'pinia' +import { computed, ref, watch } from 'vue' + +import { clientId } from '../lib/clientId' +import { graphql } from '../lib/graphql' +import { useExerciseStore } from './exercise' +import { useSettingsStore } from './settings' + +// The rest of the deck's translations into one language, in one request. The +// ids go in a variable rather than the document: 200 of them written inline +// would be over the API's token limit on their own. +const TRANSLATIONS_QUERY = ` + query Translations($ids: [ID!], $language: String!) { + exercises(filters: { id: { inList: $ids } }) { + id + translation(language: $language) { id text origin upVotes downVotes } + } + } +` + +const RATE_MUTATION = ` + mutation Rate($id: ID!, $clientId: String!, $value: TranslationRatingValue!) { + rateTranslation(translationId: $id, clientId: $clientId, value: $value) { + id text origin upVotes downVotes + } + } +` + +const PROPOSE_MUTATION = ` + mutation Propose($exerciseId: ID!, $language: String!, $text: String!, $clientId: String!) { + proposeTranslation(exerciseId: $exerciseId, language: $language, text: $text, clientId: $clientId) { + accepted + } + } +` + +// The refusals the API names that a learner can act on. Anything else — a +// network failure, a server error, a code added later — gets the generic +// message rather than none. +const KNOWN_ERRORS = new Set(['BLANK', 'TOO_LONG', 'DUPLICATE', 'RATE_LIMITED']) + +// The longest translation the API stores. The form stops there rather than +// letting the learner write something the server will refuse. +export const MAX_TRANSLATION_LENGTH = 500 + +/** The i18n key describing why a write failed. */ +export function errorKey(error) { + return KNOWN_ERRORS.has(error?.code) + ? `translation.errors.${error.code}` + : 'translation.errors.generic' +} + +const key = (exerciseId, language) => `${exerciseId}:${language}` + +/** + * The translation shown with the current exercise, and the learner's two ways + * of improving it. + * + * Its own store rather than more of the exercise store: nothing about checking + * an answer depends on it, and a failure here must never reach practice. It + * follows the exercise store's deck and the settings, fetching whatever the + * rest of the deck is missing in the language being shown, so advancing to the + * next exercise never waits on a request. + */ +export const useTranslationStore = defineStore('translation', () => { + const exercise = useExerciseStore() + const settings = useSettingsStore() + + // `${exerciseId}:${language}` -> the translation, or null for "there is none". + // Absent means not asked for yet. + const translations = ref({}) + // Same keys -> 'loading' | 'error', while a request is out or after it failed. + const pending = ref({}) + // Translation id -> 'UP' | 'DOWN': what this browser has said this visit. + // Not persisted — the server holds the rating, and nothing reads it back. + const ratings = ref({}) + + const language = computed(() => settings.interfaceLanguage) + + // Exercises carry no language of their own yet; the one they are practised + // in is the one their keyboard is for. A translation into that same language + // would say the sentence again. + const sourceLanguage = computed(() => exercise.layout.language) + + const active = computed( + () => settings.showTranslation && language.value !== sourceLanguage.value, + ) + + const currentKey = computed(() => + exercise.current ? key(exercise.current.id, language.value) : null, + ) + + /** 'loading' | 'error' | 'none' | 'ready' for the exercise on screen. */ + const state = computed(() => { + const k = currentKey.value + if (!k) return 'loading' + if (k in translations.value) return translations.value[k] ? 'ready' : 'none' + return pending.value[k] === 'error' ? 'error' : 'loading' + }) + + const current = computed(() => (state.value === 'ready' ? translations.value[currentKey.value] : null)) + + const currentRating = computed(() => (current.value ? ratings.value[current.value.id] : undefined)) + + async function fetchMissing() { + if (!active.value || !exercise.deck.length) return + + const lang = language.value + const ids = exercise.deck + .slice(exercise.index) + .map((item) => item.id) + .filter((id) => !(key(id, lang) in translations.value) && pending.value[key(id, lang)] !== 'loading') + if (!ids.length) return + + for (const id of ids) pending.value[key(id, lang)] = 'loading' + + try { + const data = await graphql(TRANSLATIONS_QUERY, { ids, language: lang }) + const found = new Map(data.exercises.map((item) => [item.id, item.translation])) + for (const id of ids) { + translations.value[key(id, lang)] = found.get(id) ?? null + delete pending.value[key(id, lang)] + } + } catch (error) { + console.error('Could not load translations:', error) + for (const id of ids) pending.value[key(id, lang)] = 'error' + } + } + + // A new deck, the setting turned on, or another interface language: each + // leaves translations to fetch, and each is answered with one request. + watch([active, language, () => exercise.deck], fetchMissing, { immediate: true }) + + /** + * Rate the shown translation. Choosing the rating already given withdraws + * it. Returns null on success, or the i18n key of what went wrong. + */ + async function rate(value) { + const translation = current.value + if (!translation) return null + + const previous = ratings.value[translation.id] + const sent = previous === value ? 'NONE' : value + + try { + const data = await graphql(RATE_MUTATION, { + id: translation.id, + clientId: clientId(), + value: sent, + }) + // Every cached copy of this translation, whichever exercise key holds it. + for (const [k, cached] of Object.entries(translations.value)) { + if (cached?.id === translation.id) translations.value[k] = data.rateTranslation + } + if (sent === 'NONE') delete ratings.value[translation.id] + else ratings.value[translation.id] = sent + return null + } catch (error) { + return errorKey(error) + } + } + + /** + * Suggest a translation of the current exercise into the interface language. + * Returns null once the server has it (it waits for review), or the i18n key + * of why it was refused. The obvious refusals are caught here first, so they + * cost no request. + */ + async function propose(text) { + const trimmed = text.trim() + if (!trimmed) return 'translation.errors.BLANK' + if (trimmed.length > MAX_TRANSLATION_LENGTH) return 'translation.errors.TOO_LONG' + if (current.value && trimmed === current.value.text) return 'translation.errors.DUPLICATE' + if (!exercise.current) return 'translation.errors.generic' + + try { + await graphql(PROPOSE_MUTATION, { + exerciseId: exercise.current.id, + language: language.value, + text: trimmed, + clientId: clientId(), + }) + return null + } catch (error) { + return errorKey(error) + } + } + + return { + translations, + ratings, + active, + state, + current, + currentRating, + language, + fetchMissing, + rate, + propose, + } +})