Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion apps/server/src/services/server-move/freeze.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@ import { serverMovingError } from "./freeze-state.js";

const READ_METHODS: ReadonlySet<string> = new Set(["GET", "HEAD", "OPTIONS"]);
const SERVER_MOVE_ROUTE_PATTERN = /^\/api\/v1\/server\/move(?:\/|$)/u;
const CONNECT_ACCOUNT_STATUS_RPC_PATHS: ReadonlySet<string> = new Set(
[
"bb-account.v1.status",
"bb-account.v1.waitForStatusChange",
"bb-account.v1.connectCredential",
].map((method) => `/api/v1/plugins/bb-account/rpc/${method}`),
);

export interface ServerMoveFreezeState {
isFrozen(): boolean;
Expand All @@ -12,7 +19,11 @@ export function isServerMoveFreezeExempt(
method: string,
path: string,
): boolean {
return READ_METHODS.has(method) || SERVER_MOVE_ROUTE_PATTERN.test(path);
return (
READ_METHODS.has(method) ||
SERVER_MOVE_ROUTE_PATTERN.test(path) ||
CONNECT_ACCOUNT_STATUS_RPC_PATHS.has(path)
);
}

export function serverMoveFreezeMiddleware(
Expand Down
75 changes: 74 additions & 1 deletion apps/server/test/server-move/coordinator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@ import {
} from "@bb/server-archive";
import type { ServerMoveStatus } from "@bb/server-contract";
import { createDeferredPromise } from "@bb/test-helpers";
import { Hono } from "hono";
import { describe, expect, it, vi } from "vitest";
import { createServerErrorHandler } from "../../src/errors.js";
import { createServerMoveCoordinator } from "../../src/services/server-move/coordinator.js";
import { serverMoveFreezeMiddleware } from "../../src/services/server-move/freeze.js";
import {
isServerMoveFrozen,
isServerMoveSnapshotFenced,
Expand All @@ -32,7 +35,11 @@ import {
type FakeDaemonReply,
} from "../helpers/server-move.js";
import { seedHost, seedPrimaryHost } from "../helpers/seed.js";
import { withTestHarness, type TestAppHarness } from "../helpers/test-app.js";
import {
testLogger,
withTestHarness,
type TestAppHarness,
} from "../helpers/test-app.js";

const OLD = "host-old";
const NEW = "host-new";
Expand Down Expand Up @@ -634,6 +641,72 @@ describe("server move coordinator", () => {
).toEqual({ serverUrl: CONNECT_URL, serverHeaders: grantHeaders });
}));

it("keeps bb connect's bb account status reads open while a bb connect move is frozen", () =>
withTestHarness(async (harness) => {
seedTopology(harness);
const base = createTestServerMoveEnvironment(harness);
const { events } = base;
const accountRpc = new Hono();
accountRpc.onError(createServerErrorHandler(testLogger));
accountRpc.use(
"/api/v1/*",
serverMoveFreezeMiddleware({ isFrozen: () => coordinator.isFrozen() }),
);
accountRpc.post("/api/v1/plugins/:id/rpc/:method", (context) =>
context.json({ ok: true, result: null }),
);
const whileFrozen: Array<[string, boolean, number]> = [];
const coordinator = createServerMoveCoordinator({
...base.environment,
exportArchive: async (args) => {
for (const method of [
"bb-account.v1.waitForStatusChange",
"bb-account.v1.status",
"bb-account.v1.connectCredential",
"bb-account.v1.fetch",
]) {
const response = await accountRpc.request(
`/api/v1/plugins/bb-account/rpc/${method}`,
{ method: "POST", body: JSON.stringify({ afterRevision: 1 }) },
);
whileFrozen.push([method, coordinator.isFrozen(), response.status]);
}
return base.environment.exportArchive(args);
},
resolveMode: async () => ({
mode: "connect",
connectHandle: "laptop",
serverUrl: CONNECT_URL,
}),
resolveServerHostGrant: async () => ({
serverUrl: CONNECT_URL,
headers: { "x-bb-connect-machine": "bbcm_laptop" },
}),
});
registerFakeDaemon(harness, { events, hostId: OLD, handle: probeReply });
registerFakeDaemon(harness, {
events,
hostId: WORKER,
handle: probeReply,
});
registerFakeDaemon(harness, {
events,
hostId: NEW,
handle: targetReply,
});

await coordinator.start({ ...START_DIRECT, serverUrl: null });
await expect.poll(() => events.includes("retire")).toBe(true);

expect(whileFrozen).toEqual([
["bb-account.v1.waitForStatusChange", true, 200],
["bb-account.v1.status", true, 200],
["bb-account.v1.connectCredential", true, 200],
["bb-account.v1.fetch", true, 503],
]);
expect(coordinator.getStatus()?.state).toBe("completed");
}));

it.each([
{
name: "the target inspect during update-target",
Expand Down
64 changes: 63 additions & 1 deletion apps/server/test/server-move/freeze.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import { Hono } from "hono";
import { describe, expect, it } from "vitest";
import { createServerErrorHandler } from "../../src/errors.js";
import { serverMoveFreezeMiddleware } from "../../src/services/server-move/freeze.js";
import {
isServerMoveFreezeExempt,
serverMoveFreezeMiddleware,
} from "../../src/services/server-move/freeze.js";
import { readJson } from "../helpers/json.js";
import { testLogger } from "../helpers/test-app.js";

const BB_ACCOUNT_RPC = "/api/v1/plugins/bb-account/rpc";

function frozenApp(state: { frozen: boolean }): Hono {
const app = new Hono();
app.onError(createServerErrorHandler(testLogger));
Expand Down Expand Up @@ -55,4 +60,61 @@ describe("server move freeze middleware", () => {
(await app.request("/api/v1/threads", { method: "POST" })).status,
).toBe(200);
});

it("keeps bb connect's bb account status reads open while frozen so the tunnel stays up", async () => {
const app = frozenApp({ frozen: true });

for (const method of [
"bb-account.v1.status",
"bb-account.v1.waitForStatusChange",
"bb-account.v1.connectCredential",
]) {
const response = await app.request(`${BB_ACCOUNT_RPC}/${method}`, {
method: "POST",
body: JSON.stringify({ afterRevision: 3 }),
});
expect(response.status, method).toBe(200);
}
for (const method of [
"bb-account.v1.fetch",
"bb-account.v1.adoptConnectCredential",
"bb-account.v1.confirmRefusedCredential",
]) {
const response = await app.request(`${BB_ACCOUNT_RPC}/${method}`, {
method: "POST",
});
expect(response.status, method).toBe(503);
expect(await readJson(response)).toMatchObject({
code: "server_moving",
});
}
});

it("exempts only the read-only bb account RPCs that bb connect polls", () => {
for (const path of [
`${BB_ACCOUNT_RPC}/bb-account.v1.status`,
`${BB_ACCOUNT_RPC}/bb-account.v1.waitForStatusChange`,
`${BB_ACCOUNT_RPC}/bb-account.v1.connectCredential`,
]) {
expect(isServerMoveFreezeExempt("POST", path), path).toBe(true);
}
for (const path of [
`${BB_ACCOUNT_RPC}/bb-account.v1.fetch`,
`${BB_ACCOUNT_RPC}/bb-account.v1.adoptConnectCredential`,
`${BB_ACCOUNT_RPC}/bb-account.v1.confirmRefusedCredential`,
`${BB_ACCOUNT_RPC}/login.start`,
`${BB_ACCOUNT_RPC}/redeemCode`,
`${BB_ACCOUNT_RPC}/signOut`,
`${BB_ACCOUNT_RPC}/bb-account.v1.statusX`,
`${BB_ACCOUNT_RPC}/bb-account.v1.status/extra`,
`${BB_ACCOUNT_RPC}/bb-account.v1.waitForStatusChange/`,
"/api/v1/plugins/connect/rpc/bb-account.v1.status",
"/api/v1/plugins/bb-account-evil/rpc/bb-account.v1.status",
"/api/v1/plugins/bb-account/cli",
`/prefix${BB_ACCOUNT_RPC}/bb-account.v1.status`,
"/api/v1/threads",
]) {
expect(isServerMoveFreezeExempt("POST", path), path).toBe(false);
}
});
});
5 changes: 4 additions & 1 deletion docs/server-move-plan.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,10 @@ All refuse requests authenticated by a machine credential.
the imported files it created.
- **At the freeze:** running turns stop, plugin schedules pause, and every
plugin except bb connect is suspended before the target is updated, so
nothing writes during the update or after the snapshot. Suspending and the
nothing writes during the update or after the snapshot. The read-only bb
account RPCs that bb connect polls (`bb-account.v1.status`,
`waitForStatusChange`, and `connectCredential`) stay available while frozen,
so the tunnel stays up. Suspending and the
later plugin shutdown are each time-boxed at 30 seconds; on expiry the move
logs a warning and continues, and retirement's forced exit covers a hung
tunnel. A failed, cancelled, or abandoned move lifts the freeze at once, so
Expand Down
Loading