diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7dbbea..83e7cb92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - 槽位种子页默认锁定:需确认解锁后才能改配置,播种前再次确认;时区/语言提供 en、cn、jp 基础选项。时区跟随当前 SOCKS5(空/非手动钉住)时展示出口时区,播种经 `timezone_follow_proxy` 写入;可改固定 IANA 时区或改回跟随。TZ/LANG/LC_ALL 仍不经 `extra_env` 覆盖契约键。 - 槽位种子页:`settings.json` 的 env 行与 `theme` 可直接编辑并随播种写入;`autoUpdates`、`grove_enabled` 只读且播种时固定为 `false`;TZ/LANG/LC_ALL、遥测 kill-switch、Bedrock/Vertex、gateway hint 等同理只读(播种由契约重写)。缺 `CLAUDE_CODE_ATTRIBUTION_HEADER` 时默认 `0`。调试上传默认关,独立 query 拉最近 40 条上传,切换开关不重置未保存编辑;Go sidecar 写 `vms//run/telemetry-upload.log`,面板 `GET seed-settings?debug=1` 返回 `telemetry_log`。 - 槽位种子页按当前槽 `settings.json` 列出真实配置;`claude.setting` 可编辑播种;缺失文件返回 `null`、空文件返回空串,不用策略默认冒充槽内文件;遥测开关与 sidecar 进程状态分开展示,保存不因遥测进程刷新失败而 502。 +- 集群槽 legacy kin-worker / kin-egress 容器可一次性迁到持久化 `/opt/kin-live` 挂载:旧容器重命名为 `*-legacy-` 保留作回滚,替换容器复用原镜像与网络身份;rename 后先从 user-defined 网 disconnect 释放 IP/别名再 create;KVM 已挂 `/opt/kin-guest/opt/kin-live` 不再误判 legacy;`scripts/migrate-legacy-cluster-slots.mjs` 与 fleet `action=telemetry` 同路径;迁移时保留 CapDrop/CapAdd 等 HostConfig,避免 canReuse 后 egress 权限缩水。(#605) - fleet 遥测刷新在 kin-egress 二进制被替换(含 `(deleted)` 旧进程与脏 pidfile)时,会按 egress 配置扫停全部 helper、等待 `/proc/` 消失后再绑定新二进制;同一 SOCKS 出口的 stop/start 与 `ensureProxyEgress` 共用 per-proxy 锁,避免并发 telemetry 互相 SIGTERM。(#604) - 面板:侧栏「GitHub / 文档」两枚链接不再折行,字体偏宽时「文档」不会逐字竖排,空间不够先缩箭头再省略文字;VM 列表搜索框关闭浏览器自动填充,不再被回填成登录邮箱。 - 集群页「扩展节点」新增 CPU / 内存 / 容器三列:健康检查(每 30 秒)经同一条 SSH 连接读节点 `/proc` 和 cgroup,口径同本机宿主列;容器 = 节点上全部 Docker 容器内存合计(扣除 inactive_file,同 `docker stats`)。修复本机宿主列 CPU 最高只显示 10%(`Math.min(100, …×1000)/10` 写错上限)。链路列不再折行。 diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index cd440e33..659c63cb 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -32,6 +32,16 @@ reporting success. A failed service restart is reported per slot so the old VM identity remains intact and can be rolled back by republishing the previous binary. +For a controlled rollout, run `scripts/migrate-legacy-cluster-slots.mjs` in +the control-plane container. Pass VM IDs after the root and concurrency +arguments to limit the rollout. It performs the same migration used by the +one-click installer: legacy slot and egress containers are stopped, renamed to +`*-legacy-`, and kept as stopped rollback copies; replacement +containers use the persistent `/opt/kin-live` mount. Container IDs therefore +change, but the VM, credentials, `vms/`, routing, and network identity do not. +After this one-time migration, subsequent updates atomically replace only the +Go binaries in `slot-bin` and restart the service processes. + ## Upload audit log (debug) Each successful or failed `PostBatch` / GrowthBook eval append one line to diff --git a/scripts/migrate-legacy-cluster-slots.mjs b/scripts/migrate-legacy-cluster-slots.mjs new file mode 100644 index 00000000..3f5a275d --- /dev/null +++ b/scripts/migrate-legacy-cluster-slots.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +/** + * Migrate legacy cluster slot containers to the persistent Go-service mount. + * This never rebuilds a VM: the old container is renamed and retained as a + * stopped rollback copy, while the replacement reuses its image and settings. + */ +import { runFleetUpdate } from '../src/lib/vm/fleet-update.mjs' + +const root = process.argv[2] || process.env.KIN_PROJECT_ROOT || '/opt/vm2api' +const concurrency = Number(process.argv[3] || process.env.KIN_FLEET_CONCURRENCY || 4) +const ids = process.argv + .slice(4) + .flatMap((value) => String(value).split(',')) + .map((value) => value.trim()) + .filter(Boolean) + +const report = await runFleetUpdate(root, { + action: 'telemetry', + concurrency, + ids: ids.length ? ids : null, +}) +console.log(JSON.stringify(report)) +if (report.failed_count) process.exitCode = 1 diff --git a/scripts/panel-upgrade.sh b/scripts/panel-upgrade.sh index 69c53879..65fa1bb1 100755 --- a/scripts/panel-upgrade.sh +++ b/scripts/panel-upgrade.sh @@ -139,7 +139,11 @@ while [ "$attempt" -lt 45 ]; do if probe verify "$TAG" "$BASELINE"; then [ -d .git ] || printf '%s\n' "${TAG#v}" > VERSION # Replace the Go worker/egress payloads in-place; no VM/container rebuild. - if ! docker exec "$CONTAINER" node /opt/vm2api/scripts/fleet-services.mjs; then + # Older control-plane images may not carry the helper yet; retry on the + # next image update instead of invalidating an otherwise verified upgrade. + if ! docker exec "$CONTAINER" test -f /opt/vm2api/scripts/fleet-services.mjs >/dev/null 2>&1; then + echo "Go service refresh helper is not present in the control plane; retry after the next image update" >&2 + elif ! docker exec "$CONTAINER" node /opt/vm2api/scripts/fleet-services.mjs; then echo "Go service-only update failed; inspect the per-slot report" >&2 exit 1 fi diff --git a/src/lib/cluster/docker-remote.mjs b/src/lib/cluster/docker-remote.mjs index c53849a5..11ce46ba 100644 --- a/src/lib/cluster/docker-remote.mjs +++ b/src/lib/cluster/docker-remote.mjs @@ -328,6 +328,16 @@ export async function removeContainer(connect, id) { await dockerJson(connect, { method: 'DELETE', path: `${containerPath(id)}?force=1&v=1`, timeoutMs: 60_000 }, [204]) } +/** Preserve the container object for a rollback instead of deleting it. */ +export async function renameContainer(connect, id, name) { + if (!NAME_RE.test(String(name || ''))) throw new ClusterError(400, 'invalid_name', '容器名不合法') + await dockerJson( + connect, + { method: 'POST', path: `${containerPath(id)}/rename?name=${encodeURIComponent(name)}`, timeoutMs: 30_000 }, + [204], + ) +} + export async function containerLogs(connect, id, { tail = 200 } = {}) { const n = Math.min(5000, Math.max(1, Number.parseInt(tail, 10) || 200)) const inspect = await dockerJson(connect, { path: `${containerPath(id)}/json` }) @@ -502,6 +512,34 @@ export async function removeNetwork(connect, name) { throw new ClusterError(502, 'docker_network_failed', dockerMessage(res)) } +/** Detach a container so its static IP / aliases are free for a replacement join. */ +export async function disconnectContainerFromNetwork(connect, network, container) { + await dockerJson( + connect, + { + method: 'POST', + path: `/networks/${encodeURIComponent(network)}/disconnect`, + body: { Container: container }, + timeoutMs: 30_000, + }, + [200, 204, 404], + ) +} + +/** Re-attach a container with the endpoint it had before a failed migration rollback. */ +export async function connectContainerToNetwork(connect, network, container, endpointConfig = {}) { + await dockerJson( + connect, + { + method: 'POST', + path: `/networks/${encodeURIComponent(network)}/connect`, + body: { Container: container, EndpointConfig: endpointConfig }, + timeoutMs: 30_000, + }, + [200, 204], + ) +} + /** * POST /build with a (gzip) tar context stream. Progress is NDJSON; a failure * after the 200 header only shows up as an `error` line. diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index 2e85879b..4a234acf 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -52,8 +52,11 @@ import { imagePresent, inspectContainerOrNull, inspectNetworkOrNull, + connectContainerToNetwork, + disconnectContainerFromNetwork, removeContainer, removeNetwork, + renameContainer, } from './docker-remote.mjs' import { awaitSlotImageBuild, @@ -78,11 +81,218 @@ import { REMOTE_WORKER_BIN, slotImageSpec } from './slot-image.mjs' /** Bump when the slot's mount layout changes; a running container with an older label is recreated. */ const SLOT_LAYOUT = '3' const LIVE_BIN = '/opt/kin-live' +const LIVE_BIN_GUEST = '/opt/kin-guest/opt/kin-live' +const LIVE_SERVICE_MOUNT_DESTINATIONS = Object.freeze([LIVE_BIN, LIVE_BIN_GUEST]) function failure(err, fallbackCode = 'remote_slot_failed') { return { ok: false, code: err?.code || fallbackCode, error: String(err?.message || err) } } +const MIGRATION_CONFIG_KEYS = Object.freeze([ + 'Hostname', + 'Domainname', + 'User', + 'AttachStdin', + 'AttachStdout', + 'AttachStderr', + 'Tty', + 'OpenStdin', + 'StdinOnce', + 'Env', + 'Cmd', + 'Image', + 'Volumes', + 'WorkingDir', + 'Entrypoint', + 'Labels', + 'StopSignal', + 'StopTimeout', + 'Shell', + 'ExposedPorts', +]) +const MIGRATION_HOST_CONFIG_KEYS = Object.freeze([ + 'Binds', + 'LogConfig', + 'NetworkMode', + 'PortBindings', + 'PublishAllPorts', + 'Privileged', + 'ReadonlyRootfs', + 'Dns', + 'DnsOptions', + 'DnsSearch', + 'ExtraHosts', + 'GroupAdd', + 'IpcMode', + 'Links', + 'Memory', + 'MemorySwap', + 'MemoryReservation', + 'MemorySwappiness', + 'NanoCpus', + 'CpuShares', + 'CpuQuota', + 'CpuPeriod', + 'CpusetCpus', + 'CpusetMems', + 'PidsLimit', + 'RestartPolicy', + 'CapAdd', + 'CapDrop', + 'SecurityOpt', + 'ShmSize', + 'UTSMode', + 'UsernsMode', + 'PidMode', + 'Devices', + 'DeviceCgroupRules', + 'Init', + 'OomKillDisable', + 'OomScoreAdj', + 'Runtime', + 'Sysctls', + 'Tmpfs', + 'VolumesFrom', + 'CgroupnsMode', +]) + +function bindDestination(bind) { + const parts = String(bind || '').split(':') + return parts.length >= 2 ? parts[1] : '' +} + +function replaceBindMount(binds, source, destination, removeDestinations = []) { + const remove = new Set([destination, ...removeDestinations]) + const next = (binds || []).filter((bind) => !remove.has(bindDestination(bind))) + next.push(`${source}:${destination}:ro`) + return next +} + +/** Build a replacement body from inspect data without changing VM-owned state. */ +export function legacyServiceContainerBody( + info, + { name, sourceDir, destination = LIVE_BIN, removeDestinations = [], command, layout } = {}, +) { + const body = {} + for (const key of MIGRATION_CONFIG_KEYS) { + if (info?.Config && info.Config[key] !== undefined) body[key] = structuredClone(info.Config[key]) + } + body.Image = info?.Config?.Image + body.Labels = { ...(info?.Config?.Labels || {}) } + if (layout) body.Labels['kin.vm.layout'] = SLOT_LAYOUT + if (command) body.Cmd = command + const host = info?.HostConfig || {} + body.HostConfig = {} + for (const key of MIGRATION_HOST_CONFIG_KEYS) { + if (host[key] !== undefined) body.HostConfig[key] = structuredClone(host[key]) + } + body.HostConfig.Binds = replaceBindMount(host.Binds, sourceDir, destination, removeDestinations) + const networkMode = String(host.NetworkMode || '') + const endpoint = info?.NetworkSettings?.Networks?.[networkMode] + if (endpoint && networkMode && !['host', 'none', 'bridge', 'default'].includes(networkMode)) { + body.NetworkingConfig = { + EndpointsConfig: { + [networkMode]: { + ...(endpoint.IPAddress ? { IPAMConfig: { IPv4Address: endpoint.IPAddress } } : {}), + ...(endpoint.Aliases ? { Aliases: endpoint.Aliases } : {}), + }, + }, + } + } + return body +} + +function migrationBackupName(name, stamp = Date.now()) { + const suffix = `-legacy-${stamp}` + return `${String(name).slice(0, Math.max(1, 63 - suffix.length))}${suffix}` +} + +function hasBindDestination(info, destination, source = '') { + return (info?.Mounts || []).some( + (mount) => mount.Destination === destination && mount.RW === false && (!source || mount.Source === source), + ) +} + +function hasLiveServiceMount(info, { sourceDir = '' } = {}) { + return (info?.Mounts || []).some( + (mount) => + LIVE_SERVICE_MOUNT_DESTINATIONS.includes(mount.Destination) && + mount.RW === false && + (!sourceDir || mount.Source === sourceDir), + ) +} + +function isUserDefinedNetwork(name) { + const value = String(name || '') + return value && !['host', 'none', 'bridge', 'default'].includes(value) && !value.startsWith('container:') +} + +function userDefinedNetworkNames(info) { + const names = new Set() + const mode = String(info?.HostConfig?.NetworkMode || '') + if (isUserDefinedNetwork(mode)) names.add(mode) + for (const net of Object.keys(info?.NetworkSettings?.Networks || {})) { + if (isUserDefinedNetwork(net)) names.add(net) + } + return [...names] +} + +function networkEndpointSnapshot(info) { + const endpoints = {} + for (const net of userDefinedNetworkNames(info)) { + const ep = info?.NetworkSettings?.Networks?.[net] + if (!ep) continue + const endpoint = {} + if (ep.IPAddress) endpoint.IPAMConfig = { IPv4Address: ep.IPAddress } + if (ep.Aliases?.length) endpoint.Aliases = ep.Aliases + if (Object.keys(endpoint).length) endpoints[net] = endpoint + } + return endpoints +} + +export function needsLegacyServiceMigration(info, { sourceDir = '' } = {}) { + return !!info && !hasLiveServiceMount(info, { sourceDir }) +} + +export async function migrateRemoteContainer(docker, name, body, { wasRunning, ops = {} } = {}) { + const inspect = ops.inspect || inspectContainerOrNull + const action = ops.action || containerAction + const rename = ops.rename || renameContainer + const create = ops.create || createContainerRaw + const disconnect = ops.disconnect || disconnectContainerFromNetwork + const connect = ops.connect || connectContainerToNetwork + const existing = await inspect(docker, name) + if (!existing) return { ok: true, action: 'absent', name } + const backup = migrationBackupName(name) + if (await inspect(docker, backup)) throw new Error(`legacy migration backup already exists: ${backup}`) + const oldId = existing.Id + const networks = userDefinedNetworkNames(existing) + const endpoints = networkEndpointSnapshot(existing) + if (wasRunning) await action(docker, name, 'stop', { t: 30 }) + await rename(docker, name, backup) + for (const net of networks) { + await disconnect(docker, net, oldId) + } + let createdId = null + try { + createdId = await create(docker, name, body) + if (wasRunning) await action(docker, createdId, 'start') + const next = await inspect(docker, name) + if (!next || (wasRunning && !next.State?.Running) || (!wasRunning && next.State?.Running)) { + throw new Error(`migrated container did not preserve state: ${name}`) + } + return { ok: true, action: 'migrated', name, old_id: oldId, new_id: next.Id, backup, was_running: wasRunning } + } catch (error) { + if (createdId) await rename(docker, createdId, migrationBackupName(`${name}-failed`)).catch(() => {}) + for (const [net, endpoint] of Object.entries(endpoints)) { + await connect(docker, net, backup, endpoint).catch(() => {}) + } + await rename(docker, backup, name).catch(() => {}) + if (wasRunning) await action(docker, name, 'start').catch(() => {}) + throw error + } +} + /** * Layout 1 kept `.claude` inside the guest-writable home. Carry its credential into * the host-owned `claude/` mount once, unless the new location already has one. @@ -936,13 +1146,62 @@ export async function reloadRemoteSlot(vm, projectRoot, { routing } = {}) { } /** Update a node slot's Go services without recreating its VM/container. */ +export async function migrateRemoteServiceContainers(vm, projectRoot) { + try { + const session = await nodeSession(vmNodeId(vm)) + const remoteDir = remoteSlotDir(session.host, vm.id) + const slotBin = `${remoteDir}/slot-bin` + const binaries = await pushSlotBinaries(vm, projectRoot, session) + const migrated = [] + const slotName = vm.runtime?.container || containerName(vm.id) + const slot = await inspectContainerOrNull(session.docker, slotName) + if (needsLegacyServiceMigration(slot, { sourceDir: slotBin })) { + const body = legacyServiceContainerBody(slot, { + name: slotName, + sourceDir: slotBin, + removeDestinations: ['/usr/local/bin/kin-worker'], + layout: true, + }) + migrated.push( + await migrateRemoteContainer(session.docker, slotName, body, { + wasRunning: !!slot.State?.Running, + }), + ) + } + const egressName = slotEgressNames(vm.id).egress + const egress = await inspectContainerOrNull(session.docker, egressName) + if (needsLegacyServiceMigration(egress, { sourceDir: slotBin })) { + const body = legacyServiceContainerBody(egress, { + name: egressName, + sourceDir: slotBin, + removeDestinations: ['/usr/local/bin/kin-egress'], + command: [`${LIVE_BIN}/kin-egress`, '-config', '/etc/kin-egress/egress.json'], + }) + migrated.push( + await migrateRemoteContainer(session.docker, egressName, body, { + wasRunning: !!egress.State?.Running, + }), + ) + } + return { + ok: true, + action: migrated.length ? 'migrated' : 'unchanged', + migrated, + binary_digest: binaries.digest || null, + } + } catch (err) { + return failure(err, 'legacy_service_migration_failed') + } +} + export async function restartRemoteServices(vm, projectRoot, { telemetryEnabled } = {}) { try { const session = await nodeSession(vmNodeId(vm)) const remoteDir = remoteSlotDir(session.host, vm.id) const slotDir = path.join(projectRoot, 'vms', vm.id) const name = vm.runtime?.container || containerName(vm.id) - const binaries = await pushSlotBinaries(vm, projectRoot, session) + const migration = await migrateRemoteServiceContainers(vm, projectRoot) + if (!migration.ok) return migration await pushSlotFiles(vm, slotDir, ['run', 'seed'], session) const target = { ...vm, runtime: { ...(vm.runtime || {}), container: name, type: runtimeKind(vm) } } const stop = 'pkill -TERM -x kin-worker 2>/dev/null || true' @@ -954,14 +1213,15 @@ export async function restartRemoteServices(vm, projectRoot, { telemetryEnabled const image = slotImageSpec(projectRoot, vm.kernel || 'ubuntu-24.04') const egress = await ensureRemoteEgress(session, vm, { imageRef: image.ref, - binaryDigest: binaries.digest, + binaryDigest: migration.binary_digest, }) if (!egress.ok) return { ok: false, error: egress.error, code: egress.code || 'egress_update_failed' } return { ok: true, action: telemetryEnabled === true ? 'restarted' : 'stopped', remote_dir: remoteDir, - binary_digest: binaries.digest || null, + binary_digest: migration.binary_digest, + migration: migration.migrated, } } catch (err) { return failure(err, 'service_update_failed') diff --git a/test/unit/remote-slot-migration.test.mjs b/test/unit/remote-slot-migration.test.mjs new file mode 100644 index 00000000..0b822055 --- /dev/null +++ b/test/unit/remote-slot-migration.test.mjs @@ -0,0 +1,205 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import vmContext from 'node:vm' +import { + legacyServiceContainerBody, + migrateRemoteContainer, + needsLegacyServiceMigration, +} from '../../src/lib/cluster/remote-slot.mjs' + +const oldSlot = { + Id: 'old-slot-id', + State: { Running: true }, + Config: { + Image: 'vm2api/kin-slot-ubuntu:old', + User: '1000:1000', + Cmd: ['/opt/kin/kin-kernel', '--gateway-worker', '--config', '/run/kin/kernel.json'], + Labels: { 'kin.vm': '1', 'kin.vm.layout': '2' }, + }, + HostConfig: { + Binds: ['/srv/vm-05/run:/run/kin', '/srv/vm-05/old-worker:/usr/local/bin/kin-worker'], + NetworkMode: 'kin-05-net', + ReadonlyRootfs: true, + RestartPolicy: { Name: 'unless-stopped' }, + }, + NetworkSettings: { Networks: { 'kin-05-net': { IPAddress: '172.20.0.2', Aliases: ['kin-05'] } } }, + Mounts: [{ Destination: '/run/kin', Source: '/srv/vm-05/run', RW: true }], +} + +test('legacy container body replaces old service bind and preserves slot settings', () => { + const body = legacyServiceContainerBody(oldSlot, { + sourceDir: '/srv/vm-05/slot-bin', + removeDestinations: ['/usr/local/bin/kin-worker'], + layout: true, + }) + assert.equal(body.Image, oldSlot.Config.Image) + assert.equal(body.User, oldSlot.Config.User) + assert.equal(body.Labels['kin.vm.layout'], '3') + assert.deepEqual(body.HostConfig.Binds, ['/srv/vm-05/run:/run/kin', '/srv/vm-05/slot-bin:/opt/kin-live:ro']) + assert.deepEqual(body.NetworkingConfig.EndpointsConfig['kin-05-net'].IPAMConfig, { IPv4Address: '172.20.0.2' }) + assert.equal(needsLegacyServiceMigration(oldSlot, { sourceDir: '/srv/vm-05/slot-bin' }), true) +}) + +test('legacy egress command points at the persistent service mount', () => { + const body = legacyServiceContainerBody(oldSlot, { + sourceDir: '/srv/vm-05/slot-bin', + removeDestinations: ['/usr/local/bin/kin-egress'], + command: ['/opt/kin-live/kin-egress', '-config', '/etc/kin-egress/egress.json'], + }) + assert.deepEqual(body.Cmd, ['/opt/kin-live/kin-egress', '-config', '/etc/kin-egress/egress.json']) +}) + +test('legacy container body preserves egress capability drops for canReuse', () => { + const egress = { + ...oldSlot, + HostConfig: { + ...oldSlot.HostConfig, + NetworkMode: 'host', + CapDrop: ['ALL'], + CapAdd: ['NET_ADMIN', 'NET_RAW'], + SecurityOpt: ['no-new-privileges'], + }, + } + const body = legacyServiceContainerBody(egress, { + sourceDir: '/srv/vm-05/slot-bin', + removeDestinations: ['/usr/local/bin/kin-egress'], + command: ['/opt/kin-live/kin-egress', '-config', '/etc/kin-egress/egress.json'], + }) + assert.deepEqual(body.HostConfig.CapDrop, ['ALL']) + assert.deepEqual(body.HostConfig.CapAdd, ['NET_ADMIN', 'NET_RAW']) + assert.deepEqual(body.HostConfig.SecurityOpt, ['no-new-privileges']) +}) + +test('migration keeps the old container as a stopped rollback copy', async () => { + const state = new Map([['kin-05', structuredClone(oldSlot)]]) + let sequence = 0 + const inspect = async (_docker, name) => state.get(name) || null + const action = async (_docker, name, operation) => { + const value = [...state.values()].find((item) => item.Id === name) || state.get(name) + if (operation === 'stop') value.State.Running = false + if (operation === 'start') value.State.Running = true + } + const rename = async (_docker, from, to) => { + const value = state.get(from) + state.delete(from) + state.set(to, value) + } + const create = async (_docker, name, body) => { + const id = `new-${++sequence}` + state.set(name, { + Id: id, + State: { Running: false }, + Config: body, + Mounts: [{ Destination: '/opt/kin-live', RW: false }], + }) + return id + } + const disconnect = async () => {} + const result = await migrateRemoteContainer( + {}, + 'kin-05', + legacyServiceContainerBody(oldSlot, { sourceDir: '/srv/vm-05/slot-bin', layout: true }), + { wasRunning: true, ops: { inspect, action, rename, create, disconnect } }, + ) + assert.equal(result.ok, true) + assert.equal(result.action, 'migrated') + assert.equal(state.get('kin-05').State.Running, true) + assert.equal(state.get(result.backup).State.Running, false) +}) + +test('needsLegacyServiceMigration skips KVM slots already on the guest kin-live mount', () => { + const kvmSlot = { + ...oldSlot, + HostConfig: { ...oldSlot.HostConfig, Binds: [`/srv/vm-05/slot-bin:/opt/kin-guest/opt/kin-live:ro`] }, + Mounts: [{ Destination: '/opt/kin-guest/opt/kin-live', Source: '/srv/vm-05/slot-bin', RW: false }], + } + assert.equal(needsLegacyServiceMigration(kvmSlot, { sourceDir: '/srv/vm-05/slot-bin' }), false) + const legacyWorker = { + ...oldSlot, + Mounts: [{ Destination: '/usr/local/bin/kin-worker', Source: '/srv/vm-05/old-worker', RW: false }], + } + assert.equal(needsLegacyServiceMigration(legacyWorker, { sourceDir: '/srv/vm-05/slot-bin' }), true) +}) + +test('migration disconnects the legacy container before pinning the same network IP', async () => { + const net = 'kin-05-net' + const ip = '172.20.0.2' + const occupancy = new Map([[net, { containerId: 'old-slot-id', ip }]]) + const events = [] + const slot = structuredClone(oldSlot) + const state = new Map([['kin-05', slot]]) + let sequence = 0 + const inspect = async (_docker, name) => state.get(name) || null + const action = async (_docker, name, operation) => { + const value = [...state.values()].find((item) => item.Id === name) || state.get(name) + if (!value) return + if (operation === 'stop') value.State.Running = false + if (operation === 'start') value.State.Running = true + } + const rename = async (_docker, from, to) => { + const value = state.get(from) + state.delete(from) + state.set(to, value) + } + const disconnect = async (_docker, network, containerId) => { + events.push(['disconnect', network, containerId]) + const held = occupancy.get(network) + if (held?.containerId === containerId) occupancy.delete(network) + } + const create = async (_docker, name, body) => { + events.push(['create', name]) + const pinned = body.NetworkingConfig?.EndpointsConfig?.[net]?.IPAMConfig?.IPv4Address + const held = occupancy.get(net) + if (held && held.ip === pinned) throw new Error('address already in use') + const id = `new-${++sequence}` + if (pinned) occupancy.set(net, { containerId: id, ip: pinned }) + state.set(name, { Id: id, State: { Running: false }, Config: body }) + return id + } + const body = legacyServiceContainerBody(slot, { sourceDir: '/srv/vm-05/slot-bin', layout: true }) + const result = await migrateRemoteContainer({}, 'kin-05', body, { + wasRunning: true, + ops: { inspect, action, rename, create, disconnect }, + }) + assert.equal(result.ok, true) + assert.deepEqual(events, [ + ['disconnect', net, 'old-slot-id'], + ['create', 'kin-05'], + ]) + assert.equal(occupancy.get(net)?.containerId, 'new-1') + assert.equal(occupancy.get(net)?.ip, ip) +}) + +test('remote service refresh forwards the migrated binary digest to egress', async () => { + const source = fs.readFileSync(new URL('../../src/lib/cluster/remote-slot.mjs', import.meta.url), 'utf8') + const body = source.slice( + source.indexOf('export async function restartRemoteServices('), + source.indexOf('export async function stopRemoteSlot('), + ) + const vm = { id: 'kin-05', node_id: 'node-a', runtime: { container: 'kin-05' } } + let receivedDigest + const restart = vmContext.runInNewContext(`${body.replace('export ', '')}; restartRemoteServices`, { + vmNodeId: () => vm.node_id, + nodeSession: async () => ({ host: {}, docker: {} }), + remoteSlotDir: () => '/slots/kin-05', + path, + migrateRemoteServiceContainers: async () => ({ ok: true, migrated: [], binary_digest: 'new-binary-digest' }), + pushSlotFiles: async () => {}, + runtimeKind: () => 'docker', + slotExecCmd: () => ({ Cmd: ['true'], User: '1000:1000' }), + execDetached: async () => {}, + setTimeout: (done) => done(), + slotImageSpec: () => ({ ref: 'slot-image' }), + ensureRemoteEgress: async (_session, _vm, { binaryDigest }) => { + receivedDigest = binaryDigest + return { ok: true } + }, + failure: (error) => ({ ok: false, error: error.message }), + }) + const result = await restart(vm, '/project', { telemetryEnabled: false }) + assert.equal(result.ok, true, result.error) + assert.equal(result.binary_digest, 'new-binary-digest') + assert.equal(receivedDigest, 'new-binary-digest') +})