diff --git a/package-lock.json b/package-lock.json index bdc45ae..c27bbc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "triarch-dev", - "version": "2.27.0", + "version": "2.28.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "triarch-dev", - "version": "2.27.0", + "version": "2.28.0", "dependencies": { "@anthropic-ai/sdk": "^0.98.0", "@dnd-kit/core": "^6.3.1", diff --git a/package.json b/package.json index b82da94..d7132b1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "triarch-dev", - "version": "2.27.0", + "version": "2.28.0", "private": true, "scripts": { "dev": "next dev", diff --git a/src/app/admin/modules/cron-health/page.tsx b/src/app/admin/modules/cron-health/page.tsx new file mode 100644 index 0000000..32bcc26 --- /dev/null +++ b/src/app/admin/modules/cron-health/page.tsx @@ -0,0 +1,259 @@ +import { redirect } from 'next/navigation'; +import { getServerSession } from 'next-auth'; +import { authOptions } from '@/lib/auth'; +import { getCurrentUserContext } from '@/lib/auth-context'; +import { readAllTenantCronHealth, type TenantCronResult } from '@/lib/cron-health/reader'; +import type { CronHealth, CronJobStatus } from '@/lib/cron-health/diff'; + +// /admin/modules/cron-health +// +// Central cron-health dashboard. For each known tenant we query Google Cloud +// Scheduler DIRECTLY (see src/lib/cron-health/reader.ts) and diff the actual +// jobs against the canonical EXPECTED_CRON_JOBS suite. The win over a +// tenant-push design: we catch MISSING / disabled crons even when nothing is +// running (today Revolution Cyber and Eve have ZERO scheduler jobs, so they +// light up red here). +// +// Per-tenant error isolation: one project failing (or having Scheduler +// disabled / no access) renders a distinct state for that section without +// breaking the page. Auth is fail-closed (staff only); per-project queries +// fail-open (show the error state, never crash). + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +const HEALTH_DOT: Record = { + green: 'bg-emerald-400', + amber: 'bg-amber-400', + red: 'bg-red-500', +}; + +const STATE_PILL: Record = { + enabled: 'bg-emerald-900/40 text-emerald-300 border border-emerald-700/30', + paused: 'bg-amber-900/40 text-amber-300 border border-amber-700/40', + disabled: 'bg-amber-900/40 text-amber-300 border border-amber-700/40', + unknown: 'bg-zinc-800 text-zinc-400 border border-zinc-700/30', +}; + +function fmtTime(iso: string | null): string { + if (!iso) return 'never'; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return iso; + return d.toISOString().replace('T', ' ').slice(0, 19) + 'Z'; +} + +function StateBadge({ status }: { status: CronJobStatus }) { + if (!status.present) { + return ( + + missing + + ); + } + const cls = status.state ? STATE_PILL[status.state] : STATE_PILL.unknown; + return ( + + {status.state ?? 'unknown'} + + ); +} + +function tenantHealth(r: TenantCronResult): CronHealth { + if (r.state !== 'ok' || !r.diff) return 'red'; + if (r.diff.missingCount > 0 || r.diff.failingCount > 0) return 'red'; + if (r.diff.pausedOrStaleCount > 0) return 'amber'; + return 'green'; +} + +function TenantSection({ r }: { r: TenantCronResult }) { + const health = tenantHealth(r); + + return ( +
+
+
+ +
+
{r.tenant.displayName}
+
+ {r.tenant.gcpProject} · {r.tenant.location} +
+
+
+
+ {r.state === 'ok' && r.diff ? ( + + {r.diff.missingCount} missing · {r.diff.failingCount} failing ·{' '} + {r.diff.pausedOrStaleCount} paused/stale · {r.jobCount} jobs found + + ) : r.state === 'no_access' ? ( + scheduler not enabled / no access + ) : ( + query error + )} +
+
+ + {r.state !== 'ok' && ( +
+
+ {r.state === 'no_access' + ? 'Cloud Scheduler is not enabled on this project, or the platform SA lacks access.' + : 'Could not query Cloud Scheduler for this project.'} +
+ {r.error &&
{r.error}
} +
+ All expected crons are effectively dark until this is resolved (operator step in PR body). +
+
+ )} + + {r.state === 'ok' && r.diff && ( +
+ + + + + + + + + + + + + {r.diff.jobs.map((j) => ( + + + + + + + + + ))} + +
Expected jobStateScheduleLast attemptLast statusHealth
+ {j.key} + {j.matchedJobName && j.matchedJobName !== j.key && ( +
+ matched: {j.matchedJobName} +
+ )} + 		+ + + {j.schedule ?? '—'} + + {fmtTime(j.lastAttemptTime)} + + {!j.present ? ( + — + ) : j.lastRunOk === null ? ( + no run yet + ) : j.lastRunOk ? ( + success + ) : ( + fail + )} + + + + {j.detail} + +
+ + {r.diff.extraJobs.length > 0 && ( +
+ Extra jobs (not in expected suite):{' '} + {r.diff.extraJobs.join(', ')} +
+ )} +
+ )} +
+ ); +} + +export default async function CronHealthDashboard() { + const session = await getServerSession(authOptions); + const ctx = await getCurrentUserContext(session); + if (!ctx?.isStaff) { + redirect('/login'); + } + + const results = await readAllTenantCronHealth(); + + const darkTenants = results.filter((r) => tenantHealth(r) === 'red'); + const amberTenants = results.filter((r) => tenantHealth(r) === 'amber'); + + return ( +
+
+
+

Cron Health — Cross-Tenant

+

+ Per-tenant scheduled-job health, read live from Google Cloud Scheduler. Each tenant's + actual jobs are diffed against the canonical expected cron suite, so this catches + missing or disabled crons even when nothing is running. Green means enabled with a + recent success, amber means paused or stale, red means missing, failing, or scheduler + not enabled. +

+
+ + {/* Top summary */} +
+
+
Tenants
+
{results.length}
+
+
0 ? 'border-red-700/40 bg-red-950/30' : 'border-zinc-800 bg-zinc-900' + }`} + > +
Tenants with dark crons
+
0 ? 'text-red-300' : 'text-emerald-300' + }`} + > + {darkTenants.length} +
+
+
+
Tenants paused/stale
+
{amberTenants.length}
+
+
+ + {darkTenants.length > 0 && ( +
+
+ {darkTenants.length} tenant{darkTenants.length !== 1 ? 's have' : ' has'} missing, + failing, or dark crons. +
+
+ {darkTenants.map((r) => r.tenant.displayName).join(', ')} +
+
+ )} + +
+ {results.map((r) => ( + + ))} +
+ +
+ Source: live{' '} + + GET cloudscheduler.googleapis.com/v1/projects/{'{project}'}/locations/{'{location}'}/jobs + {' '} + per tenant, authenticated with the platform runtime service account. Expected suite and + tenant registry: src/lib/cron-health/tenants.ts. +
+
+
+ ); +} diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index 47ec62b..f0975e7 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -16,6 +16,7 @@ import { Wrench, Columns3, HeartPulse, + AlarmClock, Coins, } from 'lucide-react'; import Link from 'next/link'; @@ -126,6 +127,7 @@ const modules = [ { title: 'Work Tracker', description: 'Unified bugs & features with list + kanban', icon: Columns3, href: '/admin/modules/tracker', color: 'text-teal-400' }, { title: 'Project Tools', description: 'CI/CD, Firebase, nav template generators', icon: Wrench, href: '/admin/platform/tools', color: 'text-emerald-400' }, { title: 'LLM Provider Health', description: 'TMI provider status, live key-test & remediation runbook', icon: HeartPulse, href: '/admin/modules/llm-health', color: 'text-rose-400' }, + { title: 'Cron Health', description: 'Per-tenant scheduled-job health, read live from Cloud Scheduler', icon: AlarmClock, href: '/admin/modules/cron-health', color: 'text-rose-400' }, { title: 'LLM Usage', description: 'Cross-tenant LLM spend, usage, and key posture', icon: Coins, href: '/admin/modules/llm-usage', color: 'text-amber-400' }, { title: 'Release Logs', description: 'Track releases across all projects', icon: FileText, href: '/admin/modules/release-logs', color: 'text-blue-400' }, { title: 'Bug Reports', description: 'Triage and track bugs across projects', icon: Bug, href: '/admin/modules/bug-reports', color: 'text-red-400' }, diff --git a/src/lib/cron-health/diff.test.ts b/src/lib/cron-health/diff.test.ts new file mode 100644 index 0000000..7aab05e --- /dev/null +++ b/src/lib/cron-health/diff.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect } from 'vitest'; +import { + diffCronJobs, + jobIdFromName, + matchExpectedKey, + type SchedulerJob, +} from './diff'; +import { EXPECTED_CRON_JOBS } from './tenants'; + +// Fixed "now" so stale checks are deterministic. +const NOW = Date.parse('2026-06-17T12:00:00Z'); +const recent = new Date(NOW - 60 * 60 * 1000).toISOString(); // 1h ago +const stale = new Date(NOW - 72 * 60 * 60 * 1000).toISOString(); // 72h ago + +function job(over: Partial): SchedulerJob { + return { + name: 'projects/p/locations/us-central1/jobs/triarch-connector-sync', + schedule: '*/15 * * * *', + state: 'ENABLED', + lastAttemptTime: recent, + status: { code: 0 }, + ...over, + }; +} + +describe('jobIdFromName', () => { + it('extracts the trailing job id from a full resource name', () => { + expect(jobIdFromName('projects/p/locations/l/jobs/atlas-prod-connector-sync')).toBe( + 'atlas-prod-connector-sync', + ); + }); + it('returns the raw value when there is no slash', () => { + expect(jobIdFromName('connector-sync')).toBe('connector-sync'); + }); +}); + +describe('matchExpectedKey — loose suffix matching', () => { + it('matches exact bare names', () => { + expect(matchExpectedKey('connector-sync', 'connector-sync')).toBe(true); + }); + it('matches - prefixing', () => { + expect(matchExpectedKey('triarch-connector-sync', 'connector-sync')).toBe(true); + }); + it('matches atlas-- prefixing', () => { + expect(matchExpectedKey('atlas-prod-connector-sync', 'connector-sync')).toBe(true); + }); + it('is case-insensitive', () => { + expect(matchExpectedKey('Atlas-PROD-Connector-Sync', 'connector-sync')).toBe(true); + }); + it('does not match a different job', () => { + expect(matchExpectedKey('triarch-workflow-tick', 'connector-sync')).toBe(false); + }); + it('does not partial-match without a hyphen boundary', () => { + expect(matchExpectedKey('xconnector-sync', 'connector-sync')).toBe(false); + }); +}); + +describe('diffCronJobs — missing detection (the RC/Eve signal)', () => { + it('flags every expected job as missing when there are zero actual jobs', () => { + const d = diffCronJobs([], NOW); + expect(d.jobs).toHaveLength(EXPECTED_CRON_JOBS.length); + expect(d.missingCount).toBe(EXPECTED_CRON_JOBS.length); + expect(d.jobs.every((j) => j.present === false)).toBe(true); + expect(d.jobs.every((j) => j.health === 'red')).toBe(true); + expect(d.jobs.every((j) => j.state === null)).toBe(true); + }); + + it('flags a single missing job among present ones', () => { + const actual = EXPECTED_CRON_JOBS.filter((k) => k !== 'retention-purge').map((k) => + job({ name: `projects/p/locations/us-central1/jobs/triarch-${k}` }), + ); + const d = diffCronJobs(actual, NOW); + expect(d.missingCount).toBe(1); + const purge = d.jobs.find((j) => j.key === 'retention-purge')!; + expect(purge.present).toBe(false); + expect(purge.health).toBe('red'); + }); +}); + +describe('diffCronJobs — per-job health classification', () => { + it('green: enabled + recent success', () => { + const d = diffCronJobs( + [job({ name: 'projects/p/locations/l/jobs/triarch-connector-sync', state: 'ENABLED', lastAttemptTime: recent, status: { code: 0 } })], + NOW, + ); + const r = d.jobs.find((j) => j.key === 'connector-sync')!; + expect(r.present).toBe(true); + expect(r.state).toBe('enabled'); + expect(r.lastRunOk).toBe(true); + expect(r.health).toBe('green'); + }); + + it('amber: paused job', () => { + const d = diffCronJobs( + [job({ name: 'projects/p/locations/l/jobs/triarch-workflow-tick', state: 'PAUSED' })], + NOW, + ); + const r = d.jobs.find((j) => j.key === 'workflow-tick')!; + expect(r.state).toBe('paused'); + expect(r.health).toBe('amber'); + expect(d.pausedOrStaleCount).toBeGreaterThanOrEqual(1); + }); + + it('amber: enabled but stale last run', () => { + const d = diffCronJobs( + [job({ name: 'projects/p/locations/l/jobs/triarch-embedding-refresh', state: 'ENABLED', lastAttemptTime: stale, status: { code: 0 } })], + NOW, + ); + const r = d.jobs.find((j) => j.key === 'embedding-refresh')!; + expect(r.health).toBe('amber'); + }); + + it('amber: enabled but never ran', () => { + const d = diffCronJobs( + [job({ name: 'projects/p/locations/l/jobs/triarch-pipeline-snapshot', state: 'ENABLED', lastAttemptTime: undefined, status: undefined })], + NOW, + ); + const r = d.jobs.find((j) => j.key === 'pipeline-snapshot')!; + expect(r.lastRunOk).toBeNull(); + expect(r.health).toBe('amber'); + }); + + it('red: last run failed', () => { + const d = diffCronJobs( + [job({ name: 'projects/p/locations/l/jobs/triarch-invoice-overdue-sweep', state: 'ENABLED', lastAttemptTime: recent, status: { code: 13, message: 'boom' } })], + NOW, + ); + const r = d.jobs.find((j) => j.key === 'invoice-overdue-sweep')!; + expect(r.lastRunOk).toBe(false); + expect(r.health).toBe('red'); + expect(r.detail).toContain('boom'); + expect(d.failingCount).toBe(1); + }); +}); + +describe('diffCronJobs — extra jobs', () => { + it('surfaces actual jobs that match no expected key', () => { + const d = diffCronJobs( + [job({ name: 'projects/p/locations/l/jobs/triarch-some-bespoke-job' })], + NOW, + ); + expect(d.extraJobs).toContain('triarch-some-bespoke-job'); + }); +}); diff --git a/src/lib/cron-health/diff.ts b/src/lib/cron-health/diff.ts new file mode 100644 index 0000000..e3434d5 --- /dev/null +++ b/src/lib/cron-health/diff.ts @@ -0,0 +1,183 @@ +// src/lib/cron-health/diff.ts +// +// Pure expected-vs-actual diff logic for the cron-health dashboard. +// +// Given the canonical EXPECTED_CRON_JOBS list and the ACTUAL jobs returned by +// the Cloud Scheduler list API for one tenant project, produce a per-expected-job +// status row (including 'missing' for jobs that should exist but don't), plus a +// list of "extra" actual jobs that don't map to any expected key. +// +// This file has NO I/O and NO framework imports so it is trivially unit-tested. + +import { EXPECTED_CRON_JOBS, type ExpectedCronKey } from './tenants'; + +// ─── Cloud Scheduler shapes (only the fields we read) ─────────────────────── + +// https://cloud.google.com/scheduler/docs/reference/rest/v1/projects.locations.jobs +export interface SchedulerJob { + // Full resource name: projects/{p}/locations/{l}/jobs/{jobId} + name: string; + schedule?: string; + state?: 'ENABLED' | 'PAUSED' | 'DISABLED' | 'UPDATE_FAILED' | 'STATE_UNSPECIFIED' | string; + lastAttemptTime?: string; + // status mirrors a google.rpc.Status; code 0 / absent == last run OK. + status?: { code?: number; message?: string }; +} + +// ─── Output shapes ────────────────────────────────────────────────────────── + +export type CronHealth = 'green' | 'amber' | 'red'; + +export interface CronJobStatus { + key: ExpectedCronKey; + present: boolean; + state: 'enabled' | 'paused' | 'disabled' | 'unknown' | null; // null when missing + schedule: string | null; + lastAttemptTime: string | null; + lastRunOk: boolean | null; // null when never run or missing + health: CronHealth; + detail: string; + matchedJobName: string | null; // the actual scheduler job id we matched, if any +} + +export interface TenantDiff { + jobs: CronJobStatus[]; + extraJobs: string[]; // actual job ids that matched no expected key + missingCount: number; + failingCount: number; + pausedOrStaleCount: number; +} + +// Considered "stale" if the last attempt is older than this and no newer success. +// 48h covers daily/hourly crons with generous slack for weekend gaps. +const STALE_AFTER_MS = 48 * 60 * 60 * 1000; + +// ─── Helpers ────────────────────────────────────────────────────────────── + +/** Trailing job id from a Cloud Scheduler resource name (or the raw value). */ +export function jobIdFromName(name: string): string { + const idx = name.lastIndexOf('/'); + return idx >= 0 ? name.slice(idx + 1) : name; +} + +/** + * Loose match: an expected key matches an actual job id if the id equals the + * key, or ends with `-` (so `atlas-prod-connector-sync` and + * `triarch-connector-sync` and `connector-sync` all match 'connector-sync'). + * Comparison is case-insensitive. + */ +export function matchExpectedKey(jobId: string, key: ExpectedCronKey): boolean { + const id = jobId.toLowerCase(); + const k = key.toLowerCase(); + return id === k || id.endsWith(`-${k}`); +} + +function normalizeState(state: string | undefined): CronJobStatus['state'] { + switch ((state ?? '').toUpperCase()) { + case 'ENABLED': + return 'enabled'; + case 'PAUSED': + return 'paused'; + case 'DISABLED': + return 'disabled'; + case '': + return 'unknown'; + default: + return 'unknown'; + } +} + +function lastRunOk(job: SchedulerJob): boolean | null { + if (!job.lastAttemptTime) return null; + // google.rpc.Status: absent or code 0 == OK. + const code = job.status?.code; + return code === undefined || code === 0; +} + +function isStale(lastAttemptTime: string | null, now: number): boolean { + if (!lastAttemptTime) return true; // enabled-but-never-ran counts as stale + const t = Date.parse(lastAttemptTime); + if (Number.isNaN(t)) return true; + return now - t > STALE_AFTER_MS; +} + +// ─── Core diff ────────────────────────────────────────────────────────────── + +/** + * Diff the expected cron suite against the actual scheduler jobs for one tenant. + * `now` is injectable for deterministic tests. + */ +export function diffCronJobs(actual: SchedulerJob[], now: number = Date.now()): TenantDiff { + const actualIds = actual.map((j) => jobIdFromName(j.name)); + const matchedActualIdx = new Set(); + + const jobs: CronJobStatus[] = EXPECTED_CRON_JOBS.map((key) => { + const idx = actualIds.findIndex((id, i) => !matchedActualIdx.has(i) && matchExpectedKey(id, key)); + + if (idx === -1) { + return { + key, + present: false, + state: null, + schedule: null, + lastAttemptTime: null, + lastRunOk: null, + health: 'red' as CronHealth, + detail: 'Missing — no scheduler job matches this expected cron.', + matchedJobName: null, + }; + } + + matchedActualIdx.add(idx); + const job = actual[idx]; + const state = normalizeState(job.state); + const okFlag = lastRunOk(job); + const lastAttempt = job.lastAttemptTime ?? null; + const stale = isStale(lastAttempt, now); + + let health: CronHealth; + let detail: string; + + if (state === 'paused' || state === 'disabled') { + health = 'amber'; + detail = `Job is ${state}.`; + } else if (okFlag === false) { + health = 'red'; + detail = `Last run failed${job.status?.message ? `: ${job.status.message}` : '.'}`; + } else if (state === 'enabled' && lastAttempt && !stale) { + health = 'green'; + detail = 'Enabled, recent successful run.'; + } else if (state === 'enabled' && stale) { + health = 'amber'; + detail = lastAttempt + ? 'Enabled but last run is stale (>48h).' + : 'Enabled but has never run.'; + } else { + // unknown state, present but unclassifiable + health = 'amber'; + detail = 'Present, state could not be classified.'; + } + + return { + key, + present: true, + state, + schedule: job.schedule ?? null, + lastAttemptTime: lastAttempt, + lastRunOk: okFlag, + health, + detail, + matchedJobName: actualIds[idx], + }; + }); + + const extraJobs = actualIds.filter((_, i) => !matchedActualIdx.has(i)); + + return { + jobs, + extraJobs, + missingCount: jobs.filter((j) => !j.present).length, + failingCount: jobs.filter((j) => j.present && j.lastRunOk === false).length, + pausedOrStaleCount: jobs.filter((j) => j.present && j.health === 'amber').length, + }; +} diff --git a/src/lib/cron-health/reader.ts b/src/lib/cron-health/reader.ts new file mode 100644 index 0000000..4f90e5e --- /dev/null +++ b/src/lib/cron-health/reader.ts @@ -0,0 +1,98 @@ +// src/lib/cron-health/reader.ts +// +// Server-side Cloud Scheduler reader for the cron-health dashboard. +// +// For each tenant GCP project we call the Cloud Scheduler list API: +// GET https://cloudscheduler.googleapis.com/v1/projects/{p}/locations/{l}/jobs +// authenticated with the platform runtime SA's access token (minted by +// mintFahAccessToken — already scoped to cloud-platform, which covers Scheduler). +// +// Per-project error isolation is the whole point: if Scheduler is not enabled or +// the SA lacks access on a project, we return a DISTINCT 'no_access' result for +// that tenant (the RC/Eve signal today) WITHOUT crashing the page or affecting +// other tenants. + +import { mintFahAccessToken } from '@/lib/fah-rollout'; +import { diffCronJobs, type SchedulerJob, type TenantDiff } from './diff'; +import { CRON_HEALTH_TENANTS, type CronHealthTenant } from './tenants'; + +const SCHEDULER_API_BASE = 'https://cloudscheduler.googleapis.com/v1'; + +export type TenantQueryState = + | 'ok' + | 'no_access' // API disabled OR permission denied — scheduler is "dark" + | 'error'; // unexpected failure (network, parse, token mint) + +export interface TenantCronResult { + tenant: CronHealthTenant; + state: TenantQueryState; + diff: TenantDiff | null; + jobCount: number; + error: string | null; +} + +interface SchedulerListResponse { + jobs?: SchedulerJob[]; + nextPageToken?: string; +} + +/** Classify a non-OK Scheduler HTTP response into a tenant query state. */ +function classifyHttpError(status: number, body: string): { state: TenantQueryState; error: string } { + // 403 with SERVICE_DISABLED, or 404 on the location, both mean "scheduler not + // enabled / no access" from the operator's point of view. + if (status === 403 || status === 404) { + return { + state: 'no_access', + error: `Scheduler not enabled or no access (HTTP ${status}): ${body.slice(0, 300)}`, + }; + } + return { state: 'error', error: `Scheduler list failed (HTTP ${status}): ${body.slice(0, 300)}` }; +} + +/** Query one tenant project. Never throws — always resolves to a result. */ +export async function readTenantCronHealth( + tenant: CronHealthTenant, + now: number = Date.now(), +): Promise { + try { + const token = await mintFahAccessToken(); + + const jobs: SchedulerJob[] = []; + let pageToken: string | undefined; + + // Paginate defensively, though tenants have well under one page of jobs. + do { + const url = new URL( + `${SCHEDULER_API_BASE}/projects/${tenant.gcpProject}/locations/${tenant.location}/jobs`, + ); + url.searchParams.set('pageSize', '100'); + if (pageToken) url.searchParams.set('pageToken', pageToken); + + const res = await fetch(url.toString(), { + headers: { Authorization: `Bearer ${token}` }, + cache: 'no-store', + }); + + if (!res.ok) { + const body = await res.text(); + const { state, error } = classifyHttpError(res.status, body); + return { tenant, state, diff: null, jobCount: 0, error }; + } + + const data = (await res.json()) as SchedulerListResponse; + if (Array.isArray(data.jobs)) jobs.push(...data.jobs); + pageToken = data.nextPageToken; + } while (pageToken); + + const diff = diffCronJobs(jobs, now); + return { tenant, state: 'ok', diff, jobCount: jobs.length, error: null }; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return { tenant, state: 'error', diff: null, jobCount: 0, error: msg }; + } +} + +/** Read all registered tenants concurrently with per-tenant isolation. */ +export async function readAllTenantCronHealth(now: number = Date.now()): Promise { + return Promise.all(CRON_HEALTH_TENANTS.map((t) => readTenantCronHealth(t, now))); +} diff --git a/src/lib/cron-health/tenants.ts b/src/lib/cron-health/tenants.ts new file mode 100644 index 0000000..67e08df --- /dev/null +++ b/src/lib/cron-health/tenants.ts @@ -0,0 +1,62 @@ +// src/lib/cron-health/tenants.ts +// +// Tenant registry + canonical expected-cron suite for the central cron-health +// dashboard (/admin/modules/cron-health). +// +// The dashboard queries Google Cloud Scheduler DIRECTLY per tenant GCP project, +// so it can flag MISSING / disabled crons even when nothing is running (the +// exact gap today: Revolution Cyber and Eve currently have ZERO scheduler +// jobs). To add a tenant, append a row to CRON_HEALTH_TENANTS; to add an +// expected job, append a key to EXPECTED_CRON_JOBS. + +export interface CronHealthTenant { + slug: string; + displayName: string; + gcpProject: string; + location: string; +} + +// Known tenants. gcpProject is the GCP project that owns the tenant's Cloud +// Scheduler jobs; location is the Scheduler region (all tenants use us-central1 +// today). Extend this list as new tenants are provisioned. +export const CRON_HEALTH_TENANTS: CronHealthTenant[] = [ + { + slug: 'triarch', + displayName: 'Triarch (tenant #0)', + gcpProject: 'triarchsecurity-atlas', + location: 'us-central1', + }, + { + slug: 'revolutioncyber', + displayName: 'Revolution Cyber', + gcpProject: 'revolutioncyber-triarchcrm', + location: 'us-central1', + }, + { + slug: 'evesecurity', + displayName: 'Eve Security', + gcpProject: 'evesecurity-triarchcrm', + location: 'us-central1', + }, +]; + +// The canonical per-tenant job set. Each entry is matched LOOSELY against the +// trailing path segment / job key of a Cloud Scheduler job name, because +// tenants name jobs inconsistently (e.g. `-` or `atlas--` +// or a bare ``). See matchExpectedKey in diff.ts for the matching rule. +export const EXPECTED_CRON_JOBS = [ + 'connector-sync', + 'event-dispatcher', + 'workflow-dispatcher', + 'workflow-enroll', + 'workflow-tick', + 'workflow-enrich', + 'embedding-refresh', + 'pipeline-snapshot', + 'invoice-overdue-sweep', + 'retention-purge', + 'foundry-ledger-scan', + 'llm-usage-report', +] as const; + +export type ExpectedCronKey = (typeof EXPECTED_CRON_JOBS)[number]; diff --git a/src/lib/version.ts b/src/lib/version.ts index a23609a..02b224e 100644 --- a/src/lib/version.ts +++ b/src/lib/version.ts @@ -1 +1 @@ -export const APP_VERSION = process.env.NEXT_PUBLIC_APP_VERSION ?? 'v2.27.0'; +export const APP_VERSION = process.env.NEXT_PUBLIC_APP_VERSION ?? 'v2.28.0';