diff --git a/entrypoint.sh b/entrypoint.sh index 7c3654e6a..2d8d6aad5 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -104,6 +104,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then mkdir -p "$DATA_CACHE_DIR" fi +# Create a directory for heap snapshots. If the backend process approaches its heap +# limit (e.g., due to a memory leak), Node will dump a snapshot here (see +# --heapsnapshot-near-heap-limit in supervisord.conf) so the cause can be diagnosed +# post-mortem instead of only seeing a generic "JavaScript heap out of memory" crash. +mkdir -p "$DATA_CACHE_DIR/heap-snapshots" + # As of v5, SOURCEBOT_ENCRYPTION_KEY must be provided explicitly via an environment variable. # @see: https://docs.sourcebot.dev/docs/upgrade/v4-to-v5-guide if [ -z "$SOURCEBOT_ENCRYPTION_KEY" ]; then diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index 7beaac840..1fc4b2309 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -91,4 +91,24 @@ export class PromClient { register: this.registry, }); } + + /** + * Removes all time series associated with a given repo across every repo-scoped + * metric. The `repo` label is a dynamic, user-controlled value (repo name), and + * prom-client never forgets a label combination on its own once it's been + * observed. Without this cleanup, every distinct repo name ever seen over the + * process's lifetime stays resident in memory forever, even after the repo is + * deleted. Call this once a repo is permanently removed (e.g., after a CLEANUP + * job deletes it) to keep the registry's memory footprint bounded by the current + * set of repos rather than the historical set. + */ + public removeRepoMetrics(repoName: string) { + for (const type of ['index', 'cleanup']) { + this.activeRepoIndexJobs.remove({ repo: repoName, type }); + this.pendingRepoIndexJobs.remove({ repo: repoName, type }); + this.repoIndexJobReattemptsTotal.remove({ repo: repoName, type }); + this.repoIndexJobFailTotal.remove({ repo: repoName, type }); + this.repoIndexJobSuccessTotal.remove({ repo: repoName, type }); + } + } } \ No newline at end of file diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts index aea1291dc..0f5db1f78 100644 --- a/packages/backend/src/repoIndexManager.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -614,6 +614,10 @@ export class RepoIndexManager { where: { id: jobData.repoId }, }); + // The repo no longer exists, so drop its time series from the metrics + // registry rather than retaining them for the lifetime of the process. + this.promClient.removeRepoMetrics(repo.name); + logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); } diff --git a/supervisord.conf b/supervisord.conf index 572136331..1c5f41a73 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -31,4 +31,8 @@ autorestart=true startretries=3 stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 -redirect_stderr=true \ No newline at end of file +redirect_stderr=true +; Dump up to 3 heap snapshots to a persisted directory if the process approaches +; its heap limit, so a real memory leak (as opposed to a transient spike) can be +; diagnosed from the actual retained object graph instead of guessing from logs. +environment=NODE_OPTIONS="--heapsnapshot-near-heap-limit=3 --diagnostic-dir=%(ENV_DATA_CACHE_DIR)s/heap-snapshots" \ No newline at end of file