From 92ff09a86502f2c032d02e18debbf094ae57699c Mon Sep 17 00:00:00 2001
From: "better-stack-hq[bot]"
 <253569367+better-stack-hq[bot]@users.noreply.github.com>
Date: Fri, 31 Jul 2026 22:05:15 +0000
Subject: [PATCH] fix(backend): cap repo-index metric cardinality and add OOM
 diagnostics

Add Node's --heapsnapshot-near-heap-limit to the backend program so a
heap snapshot is captured to a persisted directory if the process
approaches its heap limit again, giving a definitive object graph
instead of only a generic "JavaScript heap out of memory" crash log.

Also fix an identified metric-cardinality leak: repo-scoped
Prometheus metrics keyed by repo name are never cleared, so every
distinct repo name observed over the process's lifetime stays
resident in the registry forever, even after the repo is deleted.
Remove those time series once a repo is actually deleted.
---
 entrypoint.sh                            |  6 ++++++
 packages/backend/src/promClient.ts       | 20 ++++++++++++++++++++
 packages/backend/src/repoIndexManager.ts |  4 ++++
 supervisord.conf                         |  6 +++++-
 4 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/entrypoint.sh b/entrypoint.sh
index 7c3654e6a..2d8d6aad5 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -104,6 +104,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
     mkdir -p "$DATA_CACHE_DIR"
 fi
 
+# Create a directory for heap snapshots. If the backend process approaches its heap
+# limit (e.g., due to a memory leak), Node will dump a snapshot here (see
+# --heapsnapshot-near-heap-limit in supervisord.conf) so the cause can be diagnosed
+# post-mortem instead of only seeing a generic "JavaScript heap out of memory" crash.
+mkdir -p "$DATA_CACHE_DIR/heap-snapshots"
+
 # As of v5, SOURCEBOT_ENCRYPTION_KEY must be provided explicitly via an environment variable.
 # @see: https://docs.sourcebot.dev/docs/upgrade/v4-to-v5-guide
 if [ -z "$SOURCEBOT_ENCRYPTION_KEY" ]; then
diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts
index 7beaac840..1fc4b2309 100644
--- a/packages/backend/src/promClient.ts
+++ b/packages/backend/src/promClient.ts
@@ -91,4 +91,24 @@ export class PromClient {
             register: this.registry,
         });
     }
+
+    /**
+     * Removes all time series associated with a given repo across every repo-scoped
+     * metric. The `repo` label is a dynamic, user-controlled value (repo name), and
+     * prom-client never forgets a label combination on its own once it's been
+     * observed. Without this cleanup, every distinct repo name ever seen over the
+     * process's lifetime stays resident in memory forever, even after the repo is
+     * deleted. Call this once a repo is permanently removed (e.g., after a CLEANUP
+     * job deletes it) to keep the registry's memory footprint bounded by the current
+     * set of repos rather than the historical set.
+     */
+    public removeRepoMetrics(repoName: string) {
+        for (const type of ['index', 'cleanup']) {
+            this.activeRepoIndexJobs.remove({ repo: repoName, type });
+            this.pendingRepoIndexJobs.remove({ repo: repoName, type });
+            this.repoIndexJobReattemptsTotal.remove({ repo: repoName, type });
+            this.repoIndexJobFailTotal.remove({ repo: repoName, type });
+            this.repoIndexJobSuccessTotal.remove({ repo: repoName, type });
+        }
+    }
 }
\ No newline at end of file
diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts
index aea1291dc..0f5db1f78 100644
--- a/packages/backend/src/repoIndexManager.ts
+++ b/packages/backend/src/repoIndexManager.ts
@@ -614,6 +614,10 @@ export class RepoIndexManager {
                     where: { id: jobData.repoId },
                 });
 
+                // The repo no longer exists, so drop its time series from the metrics
+                // registry rather than retaining them for the lifetime of the process.
+                this.promClient.removeRepoMetrics(repo.name);
+
                 logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`);
             }
 
diff --git a/supervisord.conf b/supervisord.conf
index 572136331..1c5f41a73 100644
--- a/supervisord.conf
+++ b/supervisord.conf
@@ -31,4 +31,8 @@ autorestart=true
 startretries=3
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
-redirect_stderr=true
\ No newline at end of file
+redirect_stderr=true
+; Dump up to 3 heap snapshots to a persisted directory if the process approaches
+; its heap limit, so a real memory leak (as opposed to a transient spike) can be
+; diagnosed from the actual retained object graph instead of guessing from logs.
+environment=NODE_OPTIONS="--heapsnapshot-near-heap-limit=3 --diagnostic-dir=%(ENV_DATA_CACHE_DIR)s/heap-snapshots"
\ No newline at end of file