[fix](backend): search/ingestion/reminder correctness fixes + offline RAPTOR test suite

.github/workflows/ci.yml

@@ -17,34 +17,59 @@ jobs:
         run: |
           export MBUS_URL=https://mbus.bustime.mock.mb.thething.fyi/
           export RIDE_URL=https://ride.bustime.mock.mb.thething.fyi/
-          npm start &
-          until curl localhost:3000 > /dev/null 2>&1
-          do
-            sleep 1 # waits for initial startup
+          npm start > /tmp/server.log 2>&1 &
+          # The server binds and serves 404s long before its data is ready and
+          # a bare `curl` exits 0 on a 404, so poll data endpoints with -sf and
+          # check the content we actually need.
+          wait_for() {
+            endpoint="$1"; needle="$2"; tries="$3"
+            for i in $(seq 1 "$tries"); do
+              if curl -sf "localhost:3000$endpoint" | grep -q "$needle"; then
+                echo "ready: $endpoint has $needle (after $((i * 2))s)"
+                return 0
+              fi
+              sleep 2
+            done
+            echo "::warning::timed out after $((tries * 2))s waiting for $endpoint to contain $needle"
+            tail -30 /tmp/server.log
+            return 1
+          }
+          # Stops appear once route patterns are fetched. Required: without
+          # them nothing else can be built.
+          if ! wait_for /mbus/api/v3/getAllStops stpid 150; then
+            echo "::error::server never loaded stop data"
+            exit 1
+          fi
+          # walkingCache.json is gitignored, so CI always starts cold and the
+          # server computes every stop-pair path (~10k, several minutes),
+          # persisting them in one atomic write at the end. Wait for that file:
+          # otherwise the vitest process recomputes the whole matrix in-process
+          # and the live reminder test races its own timeout. Warn-only - the
+          # tests are the authoritative verdict.
+          # ~4 min on a dev laptop; allow generous headroom for slower runners.
+          for i in $(seq 1 450); do
+            [ -f src/assets/walkingCache.json ] && { echo "walking cache ready (after $((i * 2))s)"; break; }
+            sleep 2
           done
-          sleep 10
-          until curl localhost:3000 > /dev/null 2>&1
-          do
-            sleep 1 # waits for the walking cache to populate
-          done
-          sleep 10 # waits for the graph/predictions to be built
+          [ -f src/assets/walkingCache.json ] || echo "::warning::walking cache never persisted; tests will recompute it in-process"
+          # The routing graph is rebuilt on an interval, so predictions can lag
+          # the cache slightly.
+          wait_for /mbus/api/v3/getAllPredictions stpid 60 || true
           npx vitest run test
-          # npm test
         working-directory: ${{ github.workspace }}
 
   Typecheck:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
         uses: actions/checkout@v6
-      # - name: Placeholder
-      #   run: echo hi
-      #   working-directory: ${{ github.workspace }}
       - name: NPM Install
         run: npm i
         working-directory: ${{ github.workspace }}
       - name: Typecheck
-        run: tsc --noEmit
+        # npx: use the project-pinned TypeScript, not whatever the runner
+        # image happens to ship globally.
+        run: npx tsc --noEmit
         working-directory: ${{ github.workspace }}
 
   Typedoc:
@@ -59,6 +84,9 @@ jobs:
         run: npx typedoc --entryPointStrategy expand ./src --treatWarningsAsErrors
         working-directory: ${{ github.workspace }}
       - name: Sync files
+        # Deploy only from pushes to main: fork PRs have no secrets (the step
+        # would always fail), and unreviewed branches must not publish docs.
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: SamKirkland/FTP-Deploy-Action@v4.4.0
         with:
           server: ${{ secrets.FTP_SERVER }}

package.json

@@ -7,6 +7,7 @@
     "scripts": {
         "start": "tsx src/app.ts",
         "test": "vitest run test",
+        "test:search": "vitest run test/raptor-core.test.ts test/raptor-property.test.ts test/ingestion.test.ts test/journey-plan.test.ts test/api-handlers.test.ts",
         "stress-test": "vitest run test/search-stress.test.ts",
         "docs": "typedoc --entryPointStrategy expand ./src"
     },
@@ -21,7 +22,6 @@
         "fast-xml-parser": "^5.3.2",
         "firebase-admin": "^13.6.0",
         "lru-cache": "^11.2.5",
-        "ts-array-utils": "^0.5.0",
         "tsx": "^4.11.0",
         "zod": "^4.3.6"
     },
@@ -30,6 +30,7 @@
         "@types/node": "^20.12.12",
         "@vitest/coverage-v8": "^4.1.10",
         "typedoc": "^0.28.15",
+        "typescript": "^5.9.3",
         "vitest": "^4.1.10"
     }
 }

src/app.ts

@@ -1,16 +1,25 @@
 import express from "express";
+import { existsSync } from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
 
 import mbus from "./routes/api"
 
 const app = express();
 
+// Module-relative so the server works regardless of the launch directory.
+const DOCS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../docs");
+if (!existsSync(DOCS_DIR)) {
+    console.warn(`docs/ not found at ${DOCS_DIR} — /docs will 404 until \`npm run docs\` is run`);
+}
+
 app.use(express.json());
 app.use("/mbus/api/v3", mbus);
-app.use("/docs", express.static("docs"));
+app.use("/docs", express.static(DOCS_DIR));
 
 const PORT = process.env.PORT || 3000;
 
 
 app.listen(PORT, () => {
     console.log(`Server running on port ${PORT}`);
-});
+});

src/jobs.ts

@@ -1,21 +1,52 @@
 import { updateBusPositions, initializeRoutes, rebuildGraph } from './services/graphBuilder';
 import { initializeReminders, processRideReminders, processUniversityReminders } from './services/reminder';
 
+/**
+ * Wraps an async job so overlapping runs are skipped: if the previous tick is
+ * still in flight (e.g. a slow upstream API), the new tick is dropped instead
+ * of piling up requests and letting a stale run overwrite fresher data.
+ */
+function nonOverlapping(name: string, job: () => Promise<void>): () => Promise<void> {
+    let running = false;
+    return async () => {
+        if (running) {
+            console.warn(`Job ${name} still running; skipping this tick`);
+            return;
+        }
+        running = true;
+        try {
+            await job();
+        } catch (e) {
+            console.error(`Job ${name} failed`, e);
+        } finally {
+            running = false;
+        }
+    };
+}
+
 /**
  * Starts background jobs for updating bus positions, initializing routes, and rebuilding the graph.
  */
 export function startBackgroundJobs() {
     initializeReminders();
-    initializeRoutes().then(() => {
+
+    const guardedUpdatePositions = nonOverlapping('updateBusPositions', updateBusPositions);
+    const guardedInitRoutes = nonOverlapping('initializeRoutes', initializeRoutes);
+    const guardedRebuild = nonOverlapping('rebuildGraph', rebuildGraph);
+
+    // The boot runs share the same guards as the interval ticks, so a slow
+    // boot (e.g. cold walking cache) can never overlap — and stale-overwrite —
+    // an interval run of the same job.
+    guardedInitRoutes().then(() => {
         console.log("Routes initialized. Building initial graph...");
-        rebuildGraph();
+        return guardedRebuild();
     });
 
-    setInterval(updateBusPositions, 7500);
-    setInterval(initializeRoutes, 60000);
-    setInterval(rebuildGraph, 60 * 1000);
+    setInterval(guardedUpdatePositions, 7500);
+    setInterval(guardedInitRoutes, 60000);
+    setInterval(guardedRebuild, 60 * 1000);
     setInterval(processUniversityReminders, 7500);
     setInterval(processRideReminders, 7500);
 
     console.log("Background jobs started.");
-}
+}