fix(backend): cap repo-index metric cardinality and add OOM diagnostics

entrypoint.sh

@@ -104,6 +104,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
     mkdir -p "$DATA_CACHE_DIR"
 fi
 
+# Create a directory for heap snapshots. If the backend process approaches its heap
+# limit (e.g., due to a memory leak), Node will dump a snapshot here (see
+# --heapsnapshot-near-heap-limit in supervisord.conf) so the cause can be diagnosed
+# post-mortem instead of only seeing a generic "JavaScript heap out of memory" crash.
+mkdir -p "$DATA_CACHE_DIR/heap-snapshots"
+
 # As of v5, SOURCEBOT_ENCRYPTION_KEY must be provided explicitly via an environment variable.
 # @see: https://docs.sourcebot.dev/docs/upgrade/v4-to-v5-guide
 if [ -z "$SOURCEBOT_ENCRYPTION_KEY" ]; then

packages/backend/src/promClient.ts

@@ -91,4 +91,24 @@ export class PromClient {
             register: this.registry,
         });
     }
+
+    /**
+     * Removes all time series associated with a given repo across every repo-scoped
+     * metric. The `repo` label is a dynamic, user-controlled value (repo name), and
+     * prom-client never forgets a label combination on its own once it's been
+     * observed. Without this cleanup, every distinct repo name ever seen over the
+     * process's lifetime stays resident in memory forever, even after the repo is
+     * deleted. Call this once a repo is permanently removed (e.g., after a CLEANUP
+     * job deletes it) to keep the registry's memory footprint bounded by the current
+     * set of repos rather than the historical set.
+     */
+    public removeRepoMetrics(repoName: string) {
+        for (const type of ['index', 'cleanup']) {
+            this.activeRepoIndexJobs.remove({ repo: repoName, type });
+            this.pendingRepoIndexJobs.remove({ repo: repoName, type });
+            this.repoIndexJobReattemptsTotal.remove({ repo: repoName, type });
+            this.repoIndexJobFailTotal.remove({ repo: repoName, type });
+            this.repoIndexJobSuccessTotal.remove({ repo: repoName, type });
+        }
+    }
 }

packages/backend/src/repoIndexManager.ts

@@ -614,6 +614,10 @@ export class RepoIndexManager {
                     where: { id: jobData.repoId },
                 });
 
+                // The repo no longer exists, so drop its time series from the metrics
+                // registry rather than retaining them for the lifetime of the process.
+                this.promClient.removeRepoMetrics(repo.name);
+
                 logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`);
             }
 

supervisord.conf

@@ -31,4 +31,8 @@ autorestart=true
 startretries=3
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
-redirect_stderr=true
+redirect_stderr=true
+; Dump up to 3 heap snapshots to a persisted directory if the process approaches
+; its heap limit, so a real memory leak (as opposed to a transient spike) can be
+; diagnosed from the actual retained object graph instead of guessing from logs.
+environment=NODE_OPTIONS="--heapsnapshot-near-heap-limit=3 --diagnostic-dir=%(ENV_DATA_CACHE_DIR)s/heap-snapshots"