fix(plugin-auth): re-run membership backfill when app seeding settles (#2996)

.changeset/membership-backfill-app-seeded.md

@@ -0,0 +1,22 @@
+---
+'@objectstack/spec': minor
+'@objectstack/runtime': patch
+'@objectstack/plugin-auth': patch
+---
+
+fix(plugin-auth): re-run membership backfill when app seeding settles (#2996)
+
+The ADR-0093 D6 membership backfill — the only safety net for users created
+by app seeds (raw `engine.insert` into `sys_user` bypasses better-auth's
+`user.create.after` reconciler) — ran only once on `kernel:ready`. When a seed
+bundle overruns its inline budget (`OS_INLINE_SEED_BUDGET_MS`, default 8s) it
+finishes in the background *after* `kernel:ready`, so its users stayed
+member-less in single-org `auto` mode until the next restart re-ran the backfill.
+
+`AppPlugin` now emits a new **`app:seeded`** lifecycle event when an app's inline
+seed settles (success, partial, or fallback) — carrying `{ appId, overBudget }`,
+where `overBudget: true` marks the post-`kernel:ready` background case. plugin-auth
+subscribes and re-runs the (idempotent, self-guarding, opt-out-able)
+`backfillMemberships` on that signal, closing the window without waiting for a
+restart. No behavior change when a seed completes within budget, in multi-tenant
+mode, or under `invite-only` policy; `OS_SKIP_MEMBERSHIP_BACKFILL=1` still opts out.

docs/adr/0093-tenancy-mode-and-membership-lifecycle.md

@@ -342,6 +342,16 @@ On `kernel:ready`, **single mode + `membershipPolicy: 'auto'` only**:
 - opt-out: `OS_SKIP_MEMBERSHIP_BACKFILL=1` for operators who curate
   memberships manually.
 
+The backfill **also re-runs on the app plugin's `app:seeded` hook** (#2996).
+App seeds insert `sys_user` via a raw `engine.insert`, bypassing the
+`user.create.after` reconciler, so seeded users depend on this backfill as
+their only net. A seed bundle that overruns its inline budget
+(`OS_INLINE_SEED_BUDGET_MS`) finishes in the background *after* `kernel:ready`,
+so the one-shot `kernel:ready` pass would miss its users until the next
+restart. Re-running when seeding settles closes that window. The re-run keeps
+every property above — idempotent, self-guarding (no-op under `invite-only` /
+multi-org), and disabled by the same `OS_SKIP_MEMBERSHIP_BACKFILL=1` opt-out.
+
 Multi-org backfill is **refused by design** — there is no correct guess, and a
 wrong org assignment in a tenant-isolated deployment is a data-exposure bug,
 not a convenience. Multi-org operators repair membership through the existing

packages/core/src/kernel.ts

@@ -364,10 +364,13 @@ export class ObjectKernel {
             await this.context.trigger('kernel:ready');
 
             // Phase 3.5: Trigger kernel:bootstrapped AFTER every kernel:ready
-            // handler has settled — the "all bootstrap + seed data is ready"
+            // handler has settled — the "all synchronous bootstrap has settled"
             // anchor. Reconcile/backfill work that consumes data produced by a
             // later-starting plugin's kernel:ready handler belongs here, not in
-            // kernel:ready (where handler order would race the data). See
+            // kernel:ready (where handler order would race the data). NOTE: this
+            // does NOT guarantee background app seed data has settled (an inline
+            // seed that overruns OS_INLINE_SEED_BUDGET_MS finishes later) —
+            // subscribe `app:seeded` for that. See
             // packages/spec/src/contracts/plugin-lifecycle-events.ts.
             this.logger.debug('Triggering kernel:bootstrapped hook');
             await this.context.trigger('kernel:bootstrapped');

packages/core/src/lite-kernel.ts

@@ -77,9 +77,11 @@ export class LiteKernel extends ObjectKernelBase {
 
         // Trigger ready hook (route/middleware registration phase)
         await this.triggerHook('kernel:ready');
-        // Trigger bootstrapped hook — "all bootstrap + seed data is ready"
+        // Trigger bootstrapped hook — "all synchronous bootstrap has settled"
         // anchor, strictly after every kernel:ready handler has settled and
-        // before any HTTP socket opens (see plugin-lifecycle-events.ts).
+        // before any HTTP socket opens. NOTE: does not guarantee background app
+        // seed data has settled — subscribe `app:seeded` for that
+        // (see plugin-lifecycle-events.ts).
         await this.triggerHook('kernel:bootstrapped');
         // Trigger listening hook (HTTP servers open their socket here —
         // strictly after every kernel:ready handler has completed).

packages/plugins/plugin-auth/src/auth-plugin.test.ts

@@ -1107,4 +1107,106 @@ describe('AuthPlugin', () => {
       expect(ql.tables.sys_member).toHaveLength(0);
     });
   });
+
+  // #2996 — the ADR-0093 D6 membership backfill re-runs on `app:seeded` so
+  // users inserted by an over-budget background seed (which finishes AFTER
+  // `kernel:ready` and bypasses the `user.create.after` reconciler) still get
+  // bound to the default org without waiting for the next restart.
+  describe('Membership backfill re-run on app:seeded (#2996)', () => {
+    const OLD_MULTI = process.env.OS_MULTI_ORG_ENABLED;
+    const OLD_SKIP = process.env.OS_SKIP_MEMBERSHIP_BACKFILL;
+    let hookCapture: ReturnType<typeof createHookCapture>;
+    let ql: any;
+
+    const makeQl = () => {
+      const tables: Record<string, any[]> = {
+        sys_permission_set: [{ id: 'ps_admin', name: 'admin_full_access' }],
+        sys_user_permission_set: [
+          { id: 'ups1', user_id: 'admin', permission_set_id: 'ps_admin', organization_id: null },
+        ],
+        // The platform admin already exists; the default-org bootstrap binds it
+        // as `owner` on kernel:ready. A member-less seeded user is added later.
+        sys_user: [{ id: 'admin' }],
+        sys_member: [],
+        sys_organization: [],
+      };
+      const matches = (row: any, where: any) =>
+        Object.entries(where ?? {}).every(([k, v]) => (v === null ? row[k] == null : row[k] === v));
+      return {
+        tables,
+        registerMiddleware: vi.fn(),
+        find: vi.fn(async (object: string, q: any) =>
+          (tables[object] ?? []).filter((r) => matches(r, q?.where)).slice(0, q?.limit ?? 100),
+        ),
+        insert: vi.fn(async (object: string, data: any) => {
+          (tables[object] ??= []).push(data);
+          return data;
+        }),
+      };
+    };
+
+    beforeEach(() => {
+      delete process.env.OS_MULTI_ORG_ENABLED;
+      delete process.env.OS_SKIP_MEMBERSHIP_BACKFILL;
+      hookCapture = createHookCapture();
+      ql = makeQl();
+      mockContext.hook = hookCapture.hookFn;
+      mockContext.getService = vi.fn((name: string) => {
+        if (name === 'manifest') return { register: vi.fn() };
+        if (name === 'objectql') return ql;
+        return undefined;
+      });
+    });
+
+    afterEach(() => {
+      if (OLD_MULTI === undefined) delete process.env.OS_MULTI_ORG_ENABLED;
+      else process.env.OS_MULTI_ORG_ENABLED = OLD_MULTI;
+      if (OLD_SKIP === undefined) delete process.env.OS_SKIP_MEMBERSHIP_BACKFILL;
+      else process.env.OS_SKIP_MEMBERSHIP_BACKFILL = OLD_SKIP;
+    });
+
+    const boot = async (options: any = {}) => {
+      authPlugin = new AuthPlugin({
+        secret: 'test-secret-at-least-32-chars-long',
+        baseUrl: 'http://localhost:3000',
+        ...options,
+      });
+      await authPlugin.init(mockContext);
+      await authPlugin.start(mockContext);
+    };
+
+    it('registers an app:seeded hook alongside kernel:ready', async () => {
+      await boot();
+      expect(mockContext.hook).toHaveBeenCalledWith('app:seeded', expect.any(Function));
+    });
+
+    it('binds a user seeded after kernel:ready when app:seeded fires', async () => {
+      await boot();
+      // Boot: default org created, admin bound as owner, first backfill pass
+      // sees no member-less users.
+      await hookCapture.trigger('kernel:ready');
+      expect(ql.tables.sys_member.map((m: any) => m.user_id)).toEqual(['admin']);
+
+      // A background seed inserts a member-less user AFTER kernel:ready.
+      ql.tables.sys_user.push({ id: 'seeded_u2' });
+
+      await hookCapture.trigger('app:seeded');
+
+      const seededMember = ql.tables.sys_member.find((m: any) => m.user_id === 'seeded_u2');
+      expect(seededMember).toMatchObject({ user_id: 'seeded_u2', role: 'member' });
+    });
+
+    it('OS_SKIP_MEMBERSHIP_BACKFILL=1 disables the app:seeded re-run', async () => {
+      process.env.OS_SKIP_MEMBERSHIP_BACKFILL = '1';
+      await boot();
+      // No app:seeded hook is registered when the backfill is opted out.
+      expect(mockContext.hook).not.toHaveBeenCalledWith('app:seeded', expect.any(Function));
+
+      await hookCapture.trigger('kernel:ready');
+      ql.tables.sys_user.push({ id: 'seeded_u2' });
+      await hookCapture.trigger('app:seeded');
+
+      expect(ql.tables.sys_member.find((m: any) => m.user_id === 'seeded_u2')).toBeUndefined();
+    });
+  });
 });

packages/plugins/plugin-auth/src/auth-plugin.ts

@@ -601,28 +601,43 @@ export class AuthPlugin implements Plugin {
     // Opt out entirely via OS_SKIP_MEMBERSHIP_BACKFILL=1 (operators who curate
     // memberships by hand).
     if (String(process.env.OS_SKIP_MEMBERSHIP_BACKFILL ?? '').trim() !== '1') {
-      ctx.hook('kernel:ready', async () => {
-        try {
-          const ql: any = ctx.getService<any>('objectql');
-          const tenancy = this.tenancy;
-          if (!ql || !tenancy) return;
-          const res = await backfillMemberships(ql, {
-            policy: this.options.membershipPolicy ?? 'auto',
-            resolveTargetOrg: () => tenancy.defaultOrgId(),
-            logger: ctx.logger,
-          });
-          if (res.bound > 0) {
-            ctx.logger.info(
-              `[auth] membership backfill bound ${res.bound} pre-existing member-less user(s) to the default organization (ADR-0093 D6)`,
-              res,
-            );
+      // Serialize runs so overlapping triggers (kernel:ready + one or more
+      // `app:seeded` from multiple app bundles) don't race the same scan and
+      // trip the (organization_id, user_id) unique index into warn noise.
+      let backfillChain: Promise<void> = Promise.resolve();
+      const runBackfill = (source: string): Promise<void> => {
+        backfillChain = backfillChain.then(async () => {
+          try {
+            const ql: any = ctx.getService<any>('objectql');
+            const tenancy = this.tenancy;
+            if (!ql || !tenancy) return;
+            const res = await backfillMemberships(ql, {
+              policy: this.options.membershipPolicy ?? 'auto',
+              resolveTargetOrg: () => tenancy.defaultOrgId(),
+              logger: ctx.logger,
+            });
+            if (res.bound > 0) {
+              ctx.logger.info(
+                `[auth] membership backfill (${source}) bound ${res.bound} member-less user(s) to the default organization (ADR-0093 D6)`,
+                res,
+              );
+            }
+          } catch (e) {
+            ctx.logger.warn?.('[auth] membership backfill failed', {
+              source,
+              error: (e as Error).message,
+            });
           }
-        } catch (e) {
-          ctx.logger.warn?.('[auth] membership backfill failed', {
-            error: (e as Error).message,
-          });
-        }
-      });
+        });
+        return backfillChain;
+      };
+      ctx.hook('kernel:ready', () => runBackfill('kernel:ready'));
+      // #2996: app seeds insert `sys_user` via raw engine.insert, bypassing
+      // better-auth's `user.create.after` reconciler. A seed that overruns
+      // OS_INLINE_SEED_BUDGET_MS finishes in the background AFTER kernel:ready,
+      // so its users miss the one-shot backfill above. Re-run the (idempotent)
+      // backfill when the app plugin signals seed settle.
+      ctx.hook('app:seeded', () => runBackfill('app:seeded'));
     }
 
     // Identity-source provenance for accounts created OUTSIDE better-auth's

packages/runtime/src/app-plugin.seed.test.ts

@@ -0,0 +1,125 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { AppPlugin } from './app-plugin';
+import type { PluginContext } from '@objectstack/core';
+
+/**
+ * #2996 — AppPlugin emits `app:seeded` when its inline seed settles, so
+ * reconcilers that read seeded rows (plugin-auth's ADR-0093 D6 membership
+ * backfill) can re-run for users inserted by a seed that overran the inline
+ * budget and finished in the background AFTER `kernel:ready`.
+ *
+ * These tests force the basic-insert fallback path (no `metadata` service) so
+ * the SeedLoaderService plumbing is unnecessary — the settle signalling is the
+ * same on both branches.
+ */
+describe('AppPlugin inline-seed settle signal (app:seeded, #2996)', () => {
+    const OLD_BUDGET = process.env.OS_INLINE_SEED_BUDGET_MS;
+    const OLD_MULTI = process.env.OS_MULTI_ORG_ENABLED;
+
+    let trigger: ReturnType<typeof vi.fn>;
+    let insert: ReturnType<typeof vi.fn>;
+
+    const makeContext = (overrides: Partial<PluginContext> = {}): PluginContext =>
+        ({
+            logger: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() },
+            registerService: vi.fn(),
+            getService: vi.fn((name: string) => {
+                if (name === 'objectql') return { insert };
+                // `metadata` absent → basic-insert fallback; all others absent too.
+                return undefined;
+            }),
+            getServices: vi.fn(() => new Map()),
+            hook: vi.fn(),
+            trigger,
+            ...overrides,
+        }) as unknown as PluginContext;
+
+    const bundleWithUser = () => ({
+        id: 'seed-test-app',
+        data: [{ object: 'sys_user', records: [{ id: 'seeded_u1' }] }],
+    });
+
+    beforeEach(() => {
+        delete process.env.OS_MULTI_ORG_ENABLED;
+        trigger = vi.fn();
+    });
+
+    afterEach(() => {
+        if (OLD_BUDGET === undefined) delete process.env.OS_INLINE_SEED_BUDGET_MS;
+        else process.env.OS_INLINE_SEED_BUDGET_MS = OLD_BUDGET;
+        if (OLD_MULTI === undefined) delete process.env.OS_MULTI_ORG_ENABLED;
+        else process.env.OS_MULTI_ORG_ENABLED = OLD_MULTI;
+    });
+
+    it('emits app:seeded with overBudget=true after a background seed finishes past the budget', async () => {
+        process.env.OS_INLINE_SEED_BUDGET_MS = '10';
+        // Insert resolves well after the 10ms budget, so the race yields to the
+        // budget and the seed continues in the background.
+        insert = vi.fn(() => new Promise((resolve) => setTimeout(resolve, 60)));
+        const ctx = makeContext();
+        const plugin = new AppPlugin(bundleWithUser());
+
+        await plugin.start(ctx);
+
+        // start() returned once the budget elapsed — the background seed (and
+        // thus the settle signal) has NOT fired yet.
+        expect(trigger).not.toHaveBeenCalledWith('app:seeded', expect.anything());
+
+        await vi.waitFor(() => {
+            expect(trigger).toHaveBeenCalledWith('app:seeded', {
+                appId: 'seed-test-app',
+                overBudget: true,
+            });
+        });
+        // The user was still inserted by the background seed.
+        expect(insert).toHaveBeenCalledWith('sys_user', { id: 'seeded_u1' }, expect.anything());
+    });
+
+    it('emits app:seeded with overBudget=false when the seed completes within budget', async () => {
+        process.env.OS_INLINE_SEED_BUDGET_MS = '8000';
+        insert = vi.fn(async () => undefined); // resolves immediately
+        const ctx = makeContext();
+        const plugin = new AppPlugin(bundleWithUser());
+
+        await plugin.start(ctx);
+
+        expect(trigger).toHaveBeenCalledWith('app:seeded', {
+            appId: 'seed-test-app',
+            overBudget: false,
+        });
+    });
+
+    it('does not emit app:seeded when the app has no seed datasets', async () => {
+        insert = vi.fn(async () => undefined);
+        const ctx = makeContext();
+        const plugin = new AppPlugin({ id: 'seed-test-app' });
+
+        await plugin.start(ctx);
+
+        expect(trigger).not.toHaveBeenCalledWith('app:seeded', expect.anything());
+    });
+
+    it('does not throw when the kernel context has no trigger() function', async () => {
+        process.env.OS_INLINE_SEED_BUDGET_MS = '8000';
+        insert = vi.fn(async () => undefined);
+        const ctx = makeContext({ trigger: undefined as any });
+        const plugin = new AppPlugin(bundleWithUser());
+
+        await expect(plugin.start(ctx)).resolves.toBeUndefined();
+        expect(insert).toHaveBeenCalledWith('sys_user', { id: 'seeded_u1' }, expect.anything());
+    });
+
+    it('does not run the inline seed (or emit) in multi-tenant mode', async () => {
+        process.env.OS_MULTI_ORG_ENABLED = 'true';
+        insert = vi.fn(async () => undefined);
+        const ctx = makeContext();
+        const plugin = new AppPlugin(bundleWithUser());
+
+        await plugin.start(ctx);
+
+        expect(insert).not.toHaveBeenCalled();
+        expect(trigger).not.toHaveBeenCalledWith('app:seeded', expect.anything());
+    });
+});