Improve anticheat telemetry and moderator reviews

.github/workflows/pr.yml

@@ -127,6 +127,10 @@ jobs:
         run: pnpm run typecheck
         working-directory: ./frontend
 
+      - name: Frontend telemetry tests
+        run: pnpm run test:telemetry
+        working-directory: ./frontend
+
       - name: Frontend build
         run: pnpm run build
         working-directory: ./frontend

Makefile

@@ -108,6 +108,7 @@ check-js:
 	$(DEV) pnpm --dir frontend run check
 
 check-frontend:
+	$(DEV) pnpm --dir frontend run test:telemetry
 	$(DEV) pnpm --dir frontend run typecheck
 	$(DEV) pnpm --dir frontend run build
 

docs/antibot-review.md

@@ -0,0 +1,120 @@
+# Fair play reports and evaluation
+
+`/games/:id/ml` is a heuristic review aid, not a trained ML model or a cheating
+verdict. The worker persists reports after games and can alert tournament
+moderators. Scores do not automatically ban users.
+
+## What changed after the production audit
+
+The October 27, 2026 export contained 2,222 available non-site-bot reports, with
+866 medium/high results, but no independently confirmed outcomes. It exposed
+contradictory timing counters, missing batches in cached reports, legacy Enter
+handling and inconsistent offsets. These are collection defects, not labels for
+whether a player cheated. The export and scratch analysis live under ignored
+`tmp/`; they are not application assets or test fixtures.
+
+The earlier version-5 offline experiment reduced medium/high results to 264.
+That measured alert volume only, using exported batches and cached code/history
+features. Those figures do not describe the final version-6 scorer, and do not
+establish precision, recall or a false-positive rate.
+
+Report version 6 implements these changes:
+
+- Code style, GPT-like phrases, comment ratios, missing backspaces, zero idle
+  pauses, incomplete edit coverage, browser focus loss and missing telemetry do
+  not add suspicion. Measurements can remain available for review. Incomplete
+  coverage is explicitly a collection limitation.
+- Overlapping edit rules use their maximum. Timing and blocked-paste support is
+  capped at four points, below the review threshold of five. These weights are
+  priorities, not calibrated probabilities.
+- Modern version-3 collectors record bounded individual bulk-edit evidence with
+  offsets and explanations from the corresponding editor change. A paste or
+  undo elsewhere cannot explain that edit. Vim commands, IME composition,
+  completions, indentation, undo and permitted self-paste are accounted for.
+  Client-declared origins remain untrusted hints.
+- Legacy edit counters are evaluated per collection window, with explanations
+  limited to that window. They cannot identify the source of an individual edit.
+  Server replay jumps remain review context; a client throttle does not prove
+  that code was typed within 250 ms. Server diffs exclude unchanged code between
+  separate edits.
+- Reports persist structured evidence: stable code, category, weight, message,
+  measurements and replay offset where available. Compatibility signal strings
+  remain for existing consumers.
+
+## Collection quality and competitive evidence
+
+Version 2 introduced first/last key offsets; aggregation includes pauses between
+windows. Version 3 adds collector-session IDs, batch sequence numbers and edit
+records. A database uniqueness constraint makes retries of an identified batch
+idempotent. Missing sequences and multiple sessions are limitations, never
+positive suspicion. Legacy batches without identities remain supported.
+
+Timing is checked for contradictory samples, interval ordering, pause counts,
+missing boundaries, overlaps, session changes and offsets beyond game duration.
+Unreliable timing is excluded from rhythm scoring. Missing optional fields stay
+absent; they are not silently converted into measured zeroes. Unavailable timing
+metrics display as unavailable.
+
+Competitive code and replay history stop at the first completion marker, including
+submission code that arrived before the final throttled editor update. Finished
+non-tournament give-ups also close this boundary. Later practice edits are not
+used as the competitive solution. The report stores that solution and language,
+so cached pages display the same code that was analyzed.
+
+Only complete telemetry windows ending within the known competitive duration are
+used. Duration is recorded in whole seconds; its last partial second is allowed.
+A window crossing the boundary is excluded as a whole rather than attributing its
+aggregate counters to competitive play. Excluded windows and incomplete coverage
+are shown as limitations when retained telemetry is available; an entirely
+missing competitive sample shows insufficient telemetry.
+
+Finished caches include batch count, maximum ID, game state and duration. Every
+read checks the snapshot and requires a current report for each eligible player.
+Late flushes, lower-ID transactions committing late and changes from playing to
+finished invalidate reports. Live reports recompute even without new batches.
+Writes after an analysis read are detected on the next read.
+
+## Moderator decisions and evaluation
+
+Admins and moderators can append a review on the report page with one of:
+`confirmed_assistance` (confirmed unauthorized assistance), `legitimate`, or
+`inconclusive`. A reason is required; a replay offset is optional. Records retain
+the reviewer, timestamp, detector version and report snapshot. Re-analysis does
+not overwrite decisions. Decisions are distinct from user complaints and solution
+approval; their reasons are only sent to moderators.
+
+Run the read-only evaluation command inside the dev container:
+
+```sh
+./bin/dev mix codebattle.antibot.evaluate
+```
+
+It starts only the database reader and registry, without application workers,
+and uses a read-only, repeatable-read transaction. It recomputes the current rules
+against the latest decision for each player/game in finished games,
+without persisting reports or broadcasting alerts. It reports confusion counts,
+precision, recall, false-positive rate, inconclusive/missing cases and language/
+collector cohorts. Undefined rates are null. These metrics describe the reviewed
+sample; selection bias and repeated players still matter. Without labels the
+command does not claim accuracy.
+
+For a silent historical report backfill, invoke
+`Codebattle.Game.BotDetection.analyze_and_persist(game_id)` for a bounded set of
+finished games. The worker path can broadcast tournament alerts and should not
+be used for a silent audit.
+
+## Rollout and ML
+
+Apply all additive report/review migrations before deploying the application.
+Deploy the collector and scorer together; old clients remain supported. Monitor
+missing/legacy/inconsistent telemetry separately from score and alert counts.
+Review low-scoring games as well as flagged games to measure missed detections.
+
+Build independently reviewed examples covering mobile/IME, Vim, completions,
+undo, self-paste, reconnects, slow networks and known injection/autotyper cases.
+Do not turn complaints, existing heuristic scores or practice bots into cheating
+labels. Split future ML datasets by player and reserve a later time period for
+drift evaluation. Compare candidates with these rules on the same held-out cases,
+set a moderator workload and false-positive target, and run models in shadow mode
+before using them for alerts. Keep replay review and avoid automatic bans from
+scores alone.

frontend/package.json

@@ -7,6 +7,7 @@
     "dev": "vite",
     "build": "vite build",
     "preview": "vite preview",
+    "test:telemetry": "node --test test/telemetry.test.mjs",
     "typecheck": "tsc --noEmit",
     "lint": "oxlint --deny-warnings",
     "check": "tsc --noEmit && oxlint --deny-warnings"

frontend/src/features/game/components/TopBar.tsx

@@ -165,8 +165,8 @@ export function TopBar({
         </ActionIcon>
       </Tooltip>
       {(canManage || over) && (
-        <Tooltip label={t('Bot detection (ML)')}>
-          <ActionIcon component="a" href={`/games/${game.id}/ml`} size="sm" variant="default" aria-label={t('Bot detection (ML)')}>
+        <Tooltip label={t('Fair play review')}>
+          <ActionIcon component="a" href={`/games/${game.id}/ml`} size="sm" variant="default" aria-label={t('Fair play review')}>
             <IconRobot size={14} />
           </ActionIcon>
         </Tooltip>

frontend/src/features/game/editor/telemetry.ts

@@ -24,16 +24,21 @@ export interface TelemetryEvent {
   large_insert_count?: number;
   assisted_chars?: number;
   completion_insert_count?: number;
+  command_insert_count?: number;
   paste_chars?: number;
   /** focus_return: how long the page was hidden or unfocused. */
   away_ms?: number;
 }
 
 export const WINDOW_MS = 10_000;
 export const EVENT_LIMIT = 100;
+// Version 2 records key boundaries so pauses between batches remain observable.
+export const COLLECTOR_VERSION = 3;
 
 interface TelemetryWindow {
+  edit_evidence: { offset_ms: number; inserted_chars: number; assisted_chars: number; paste_chars: number; printable_keys: number }[];
   event_count: number;
+  collector_version: number;
   window_start_offset_ms: number;
   window_end_offset_ms: number;
   lang_slug: string;
@@ -64,6 +69,7 @@ interface TelemetryWindow {
   large_insert_count: number;
   assisted_chars: number;
   completion_insert_count: number;
+  command_insert_count: number;
   paste_chars: number;
   blur_count: number;
   away_ms: number;
@@ -74,10 +80,13 @@ interface TelemetryWindow {
   max_key_delta_ms: number;
   idle_pause_over_2s_count: number;
   last_key_offset_ms: number | null;
+  first_key_offset_ms: number | null;
 }
 
 const newWindow = (event: TelemetryEvent): TelemetryWindow => ({
+  edit_evidence: [],
   event_count: 0,
+  collector_version: COLLECTOR_VERSION,
   window_start_offset_ms: event.offset_ms,
   window_end_offset_ms: event.offset_ms,
   lang_slug: event.lang_slug,
@@ -108,6 +117,7 @@ const newWindow = (event: TelemetryEvent): TelemetryWindow => ({
   large_insert_count: 0,
   assisted_chars: 0,
   completion_insert_count: 0,
+  command_insert_count: 0,
   paste_chars: 0,
   blur_count: 0,
   away_ms: 0,
@@ -118,6 +128,7 @@ const newWindow = (event: TelemetryEvent): TelemetryWindow => ({
   max_key_delta_ms: 0,
   idle_pause_over_2s_count: 0,
   last_key_offset_ms: null,
+  first_key_offset_ms: null,
 });
 
 const ARROWS = new Set(['ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight']);
@@ -132,6 +143,7 @@ export function addEvent(current: TelemetryWindow | null, event: TelemetryEvent)
 
   switch (event.type) {
     case 'keydown': {
+      if (w.first_key_offset_ms == null) w.first_key_offset_ms = event.offset_ms;
       w.key_event_count += 1;
       const modifier = event.ctrl_key || event.meta_key;
       if (typeof event.key === 'string' && event.key.length === 1 && !modifier && !event.alt_key) w.printable_key_count += 1;
@@ -167,6 +179,11 @@ export function addEvent(current: TelemetryWindow | null, event: TelemetryEvent)
       w.drop_blocked_count += 1;
       break;
     case 'content_change':
+      if ((event.inserted_chars ?? 0) >= 8) w.edit_evidence.push({
+        offset_ms: event.offset_ms, inserted_chars: event.inserted_chars ?? 0,
+        assisted_chars: event.assisted_chars ?? 0, paste_chars: event.paste_chars ?? 0,
+        printable_keys: w.last_key_offset_ms != null && event.offset_ms - w.last_key_offset_ms <= 500 ? 1 : 0,
+      });
       w.content_change_count += event.change_count ?? 0;
       w.chars_inserted += event.inserted_chars ?? 0;
       w.chars_deleted += event.deleted_chars ?? 0;
@@ -179,6 +196,7 @@ export function addEvent(current: TelemetryWindow | null, event: TelemetryEvent)
       w.large_insert_count += event.large_insert_count ?? 0;
       w.assisted_chars += event.assisted_chars ?? 0;
       w.completion_insert_count += event.completion_insert_count ?? 0;
+      w.command_insert_count += event.command_insert_count ?? 0;
       w.paste_chars += event.paste_chars ?? 0;
       break;
     case 'focus_return':
@@ -195,7 +213,7 @@ export function addEvent(current: TelemetryWindow | null, event: TelemetryEvent)
 /** The summary pushed to the server; null for an empty window. */
 export function finalize(w: TelemetryWindow | null) {
   if (!w || w.event_count <= 0) return null;
-  const { last_key_offset_ms: _last, key_delta_total_ms: total, ...rest } = w;
+  const { key_delta_total_ms: total, ...rest } = w;
   return { ...rest, avg_key_delta_ms: w.key_delta_sample_count > 0 ? Math.round(total / w.key_delta_sample_count) : 0 };
 }
 
@@ -226,16 +244,20 @@ export interface ContentChangeEvent {
 /** The `executeEdits` source of our own text syncs (language switch, reset, the server's copy). */
 export const SYNC_EDIT_SOURCE = 'cb-sync';
 
-type EditOrigin = 'sync' | 'typed' | 'completion' | 'command' | 'paste' | 'other';
+type EditOrigin = 'sync' | 'composition' | 'typed' | 'completion' | 'command' | 'paste' | 'other';
 
 const originOf = (reason: EditReason | undefined, event: ContentChangeEvent): EditOrigin => {
   if (event.isFlush) return 'sync';
   if (event.isUndoing || event.isRedoing) return 'command';
   const { source, kind, name } = reason?.metadata ?? {};
   if (source === 'setValue' || (source === 'unknown' && name === SYNC_EDIT_SOURCE)) return 'sync';
+  // Our installed Vim adapter uses executeEdits('vim') for normal-mode commands.
+  // This is an origin hint, not a trusted assertion that the player is human.
+  if (source === 'unknown' && name === 'vim') return 'command';
   if (source === 'suggest' || source === 'snippet') return 'completion';
   if (source === 'cursor') {
-    if (kind === 'type' || kind === 'compositionType' || kind === 'compositionEnd') return 'typed';
+    if (kind === 'compositionType' || kind === 'compositionEnd') return 'composition';
+    if (kind === 'type') return 'typed';
     // Tab / indent / comment toggles and the like.
     if (kind === 'executeCommand' || kind === 'executeCommands') return 'command';
     // The paste guard only lets through text copied from this editor (pasteGuard.ts).
@@ -278,6 +300,7 @@ export function contentMetrics(event: ContentChangeEvent) {
     large_insert_count: 0,
     assisted_chars: 0,
     completion_insert_count: 0,
+    command_insert_count: 0,
     paste_chars: 0,
   };
 
@@ -294,7 +317,7 @@ export function contentMetrics(event: ContentChangeEvent) {
 
     // One key, more text: auto-indent ("\n" + spaces) or an auto-closed pair.
     const keyAssisted = origin === 'typed' && (inserted <= 2 || change.text.trim() === '');
-    const assisted = keyAssisted || origin === 'completion' || origin === 'command';
+    const assisted = keyAssisted || origin === 'completion' || origin === 'command' || origin === 'composition';
     if (origin === 'paste') {
       metrics.paste_chars += inserted;
       return;
@@ -303,6 +326,7 @@ export function contentMetrics(event: ContentChangeEvent) {
       if (origin === 'typed') metrics.max_single_insert_len = Math.max(metrics.max_single_insert_len, Math.min(inserted, 1));
       metrics.assisted_chars += origin === 'typed' ? Math.max(0, inserted - 1) : inserted;
       if (origin === 'completion') metrics.completion_insert_count += 1;
+      if (origin === 'command' && inserted > 0) metrics.command_insert_count += 1;
       return;
     }
 

frontend/src/features/game/editor/useEditorTelemetry.ts

@@ -5,14 +5,16 @@ import { useLatest } from '@/shared/useLatest';
 import type { EditorTelemetry } from './CodeEditor';
 import { addEvent, EVENT_LIMIT, finalize, WINDOW_MS } from './telemetry';
 
-type Send = (summary: NonNullable<ReturnType<typeof finalize>>, lang: string) => void;
+type Send = (summary: NonNullable<ReturnType<typeof finalize>> & { collector_session: string; batch_sequence: number }, lang: string) => void;
 
 /**
  * Collects my editor's telemetry into windows (≤ 10 s / ≤ 100 events, one language each)
  * and sends each finished window. `startedAtMs` anchors the offsets (the game start).
  */
 export function useEditorTelemetry({ enabled, lang, startedAtMs, send }: { enabled: boolean; lang: string; startedAtMs: number; send: Send }) {
   const windowRef = useRef<ReturnType<typeof addEvent> | null>(null);
+  const collectorSession = useRef(crypto.randomUUID());
+  const sequence = useRef(0);
   const timer = useRef<number | null>(null);
   const sendRef = useLatest(send);
 
@@ -23,7 +25,7 @@ export function useEditorTelemetry({ enabled, lang, startedAtMs, send }: { enabl
     }
     const summary = finalize(windowRef.current);
     windowRef.current = null;
-    if (summary) sendRef.current(summary, summary.lang_slug);
+    if (summary) sendRef.current({ ...summary, collector_session: collectorSession.current, batch_sequence: sequence.current++ }, summary.lang_slug);
   }, [sendRef]);
 
   const add = useCallback(