diff --git a/CHANGELOG.md b/CHANGELOG.md index 480ac7922..3fea45fb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 未发布 + +- Docker 槽在线扩容：面板成功须 HostConfig 内存与 swap 一致；可读 cgroup 时核对 v2 `memory.max` 或 v1 `memory.limit_in_bytes`（`max`/无上限哨兵在 HostConfig 已匹配时跳过）。扩容后校验失败会 docker update 回滚至扩容前限额且不写 vm 配置。（#623，REQ-0441） + ## 1.3.201 — 2026-10-20 - 消息线程 beta 在控制面按消息 id 记住 24 小时（最多 1 万条）：同模型的 `continue` 补回上一轮 beta，并保留本轮新算出的；换模型、过期或未知 id 用现算值，不失败。create 轮对支持的模型仅凭 `thread` 带上 `mid-conversation-output-config`，continue 不再回放 effort 控件也能带上该门。命中 / 未命中 / 换模型记在 `hop_meta.thread_betas`。进程重启后 pin 丢失，按未命中处理。（#621） diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index de54e7e0f..2b6386694 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -28,7 +28,7 @@ VM2API_DB_SECRET='再一串' 不是一个父容器里多个子进程。 -槽位默认内存上限为 `512m`，在面板「规格」或 `routing.json` 的 `vm.memory` 配置（`256m` / `512m` / `1g` / `2g` / `4g` / `8g` / `16g`）。创建时可按槽覆盖，结果写在该槽 `vm.machine.memory`。已不再读取 `KIN_VM_MEMORY`。低于 1G 时，常驻 native host 与临时官方 CLI 同时运行可能 OOM；额度 API 探测本身不应额外启动推理 CLI。已有容器不会因控制面升级自动改内存，可在核对宿主余量后使用面板在线扩容，不重建容器。面板成功响应以 Docker 的内存和 swap 上限、容器内 `memory.max` 三者均匹配为准；内核每次准入按当前 `memory.max` 动态计算阈值，并按请求实际序列化大小限制在途负载。单次请求的 CLI 内存突增仍可能超过 512 MiB，准入不保证任意请求都能在此上限内运行。 +槽位默认内存上限为 `512m`，在面板「规格」或 `routing.json` 的 `vm.memory` 配置（`256m` / `512m` / `1g` / `2g` / `4g` / `8g` / `16g`）。创建时可按槽覆盖，结果写在该槽 `vm.machine.memory`。已不再读取 `KIN_VM_MEMORY`。低于 1G 时，常驻 native host 与临时官方 CLI 同时运行可能 OOM；额度 API 探测本身不应额外启动推理 CLI。已有容器不会因控制面升级自动改内存，可在核对宿主余量后使用面板在线扩容，不重建容器。面板成功须 Docker HostConfig 的 memory 与 memory-swap 均等于请求值；若容器内可读 cgroup 限额（cgroup v2 的 `memory.max`，否则 v1 的 `memory/memory.limit_in_bytes`），则须与请求一致；读不到路径或值为 `max`/无上限哨兵时，在 HostConfig 已匹配的前提下跳过 cgroup 核对。`docker update` 成功后若后续校验失败，控制面会用扩容前的 memory/swap **回滚**容器限额且**不写** vm 配置；回滚失败时 API 会提示手动核对。内核每次准入按当前 cgroup 限额动态计算阈值，并按请求实际序列化大小限制在途负载。单次请求的 CLI 内存突增仍可能超过 512 MiB，准入不保证任意请求都能在此上限内运行。 KVM 槽需要宿主 `/dev/kvm` 可读写（控制面在容器内时，用一次 `--device /dev/kvm` 的探测容器确认）。runner 降权给槽位用户跑 QEMU 时会带上 `/dev/kvm` 的设备组，属组与槽位主 GID 不同也能用。已有 KVM 槽要更新 `kin-kvm-*` 镜像并重建一次。没有 KVM 时不要在生产打开 `vm.allow_tcg`：那是 TCG 软件模拟，仅供测试。 diff --git a/src/lib/vm/docker-slot-memory.mjs b/src/lib/vm/docker-slot-memory.mjs index af293d6b4..9608dab84 100644 --- a/src/lib/vm/docker-slot-memory.mjs +++ b/src/lib/vm/docker-slot-memory.mjs @@ -55,6 +55,68 @@ function dockerExec(vm, argv, execImpl, timeout) { return execImpl(argv, { timeout, ...(env ? { env } : {}) }) } +/** cgroup v1「无上限」哨兵（与 Linux PAGE_COUNTER_MAX 同量级） */ +const CGROUP_V1_UNLIMITED = 9223372036854771712 + +export function parseCgroupMemoryLimit(raw) { + const text = String(raw ?? '').trim() + if (!text) return { kind: 'unreadable' } + if (text.toLowerCase() === 'max') return { kind: 'unlimited' } + if (/^\d+$/.test(text)) { + try { + const big = BigInt(text) + if (big >= BigInt(CGROUP_V1_UNLIMITED) - 1024n) return { kind: 'unlimited' } + const bytes = Number(big) + if (!Number.isSafeInteger(bytes) || bytes < 0) return { kind: 'unreadable' } + return { kind: 'limited', bytes } + } catch { + return { kind: 'unreadable' } + } + } + return { kind: 'unreadable' } +} + +export function readContainerCgroupMemoryLimit(vm, containerId, execImpl) { + const v2 = dockerExec(vm, ['docker', 'exec', containerId, 'cat', '/sys/fs/cgroup/memory.max'], execImpl, 10_000) + if (v2.ok) { + const parsed = parseCgroupMemoryLimit(v2.stdout) + if (parsed.kind !== 'unreadable') return parsed + } + const v1 = dockerExec( + vm, + ['docker', 'exec', containerId, 'cat', '/sys/fs/cgroup/memory/memory.limit_in_bytes'], + execImpl, + 10_000, + ) + if (v1.ok) { + const parsed = parseCgroupMemoryLimit(v1.stdout) + if (parsed.kind !== 'unreadable') return parsed + } + return { kind: 'unreadable' } +} + +function rollbackDockerMemory(vm, before, execImpl) { + const mem = before.memory + const swap = + before.swapBytes > 0 && before.swapBytes !== before.bytes ? formatDockerMemoryBytes(before.swapBytes) || mem : mem + const name = containerName(vm.id) + const r = dockerExec(vm, ['docker', 'update', '--memory', mem, '--memory-swap', swap, name], execImpl, 15_000) + return r.ok +} + +function failAfterDockerUpdate(vm, before, execImpl, failure) { + const rollbackOk = rollbackDockerMemory(vm, before, execImpl) + const rollbackNote = rollbackOk + ? '已用 docker update 回滚至扩容前限额，vm 配置未写入。' + : '自动回滚 docker update 失败，请手动核对容器 HostConfig 与面板配置是否一致。' + return { + ...failure, + message: `${failure.message} ${rollbackNote}`.trim(), + config_unchanged: true, + rollback_succeeded: rollbackOk, + } +} + export function inspectDockerContainerMemory(vm, execImpl = defaultExec) { const name = containerName(vm?.id) const r = dockerExec( @@ -140,32 +202,32 @@ export function raiseDockerSlotMemory(projectRoot, vm, requestedMemory, { routin const inspected = inspectDockerContainerMemory(vm, execImpl) if (!inspected.ok) { - return { + return failAfterDockerUpdate(vm, before, execImpl, { ok: false, code: inspected.code, message: inspected.message, requested: mem, - } + }) } const requestedBytes = memoryMiB(mem) * 1024 ** 2 if (inspected.bytes !== requestedBytes || inspected.swapBytes !== requestedBytes) { - return { + return failAfterDockerUpdate(vm, before, execImpl, { ok: false, code: 'memory_diverged', message: `容器内存或 swap 上限与请求的 ${mem} 不一致，未写入配置`, requested: mem, container_memory: inspected.memory, - } + }) } - const cgroup = dockerExec(vm, ['docker', 'exec', name, 'cat', '/sys/fs/cgroup/memory.max'], execImpl, 10_000) - if (!cgroup.ok || Number(String(cgroup.stdout || '').trim()) !== requestedBytes) { - return { + const cgroup = readContainerCgroupMemoryLimit(vm, name, execImpl) + if (cgroup.kind === 'limited' && cgroup.bytes !== requestedBytes) { + return failAfterDockerUpdate(vm, before, execImpl, { ok: false, code: 'cgroup_memory_diverged', message: '容器实际 cgroup 内存上限与请求不一致，未写入配置', requested: mem, container_memory: inspected.memory, - } + }) } const saved = patchVmRecord(projectRoot, vm.id, (record) => { @@ -178,13 +240,13 @@ export function raiseDockerSlotMemory(projectRoot, vm, requestedMemory, { routin return next }) if (!saved) { - return { + return failAfterDockerUpdate(vm, before, execImpl, { ok: false, code: 'vm_not_found', message: 'vm not found', requested: mem, container_memory: inspected.memory, - } + }) } return { diff --git a/test/unit/docker-slot-memory.test.mjs b/test/unit/docker-slot-memory.test.mjs index c66ebad10..987880b31 100644 --- a/test/unit/docker-slot-memory.test.mjs +++ b/test/unit/docker-slot-memory.test.mjs @@ -6,6 +6,7 @@ import path from 'node:path' import { formatDockerMemoryBytes, inspectDockerContainerMemory, + parseCgroupMemoryLimit, raiseDockerSlotMemory, validateDockerMemoryRaise, } from '../../src/lib/vm/docker-slot-memory.mjs' @@ -15,6 +16,13 @@ test('formatDockerMemoryBytes normalizes docker byte limits', () => { assert.equal(formatDockerMemoryBytes(1073741824), '1g') }) +test('parseCgroupMemoryLimit handles v2 max and v1 unlimited sentinel', () => { + assert.deepEqual(parseCgroupMemoryLimit('max'), { kind: 'unlimited' }) + assert.deepEqual(parseCgroupMemoryLimit('9223372036854771712'), { kind: 'unlimited' }) + assert.deepEqual(parseCgroupMemoryLimit(String(1024 ** 3)), { kind: 'limited', bytes: 1024 ** 3 }) + assert.deepEqual(parseCgroupMemoryLimit(''), { kind: 'unreadable' }) +}) + test('validateDockerMemoryRaise rejects kvm and invalid or non-increasing memory', () => { const kvm = { id: 'vm-1', runtime_type: 'kvm', machine: { memory: '512m' } } const kvmCheck = validateDockerMemoryRaise(kvm, '1g', {}) @@ -128,18 +136,28 @@ test('raiseDockerSlotMemory refuses a mismatched swap limit without persisting', tmpVmRecord(root, 'vm-04') const vm = { id: 'vm-04', runtime_type: 'docker', machine: { memory: '512m' } } let updated = false + let rollback = false const execImpl = (argv) => { if (argv.includes('update')) { - updated = true + if (argv.includes('512m') && updated) { + rollback = true + updated = false + } else { + updated = true + } return { ok: true, stdout: '' } } if (argv.includes('inspect')) { const memory = updated ? 1024 ** 3 : 512 * 1024 ** 2 - return { ok: true, stdout: `${memory} ${512 * 1024 ** 2} true` } + const swap = updated ? 512 * 1024 ** 2 : 512 * 1024 ** 2 + return { ok: true, stdout: `${memory} ${swap} true` } } return { ok: false, stderr: 'unexpected' } } - assert.equal(raiseDockerSlotMemory(root, vm, '1g', { execImpl }).code, 'memory_diverged') + const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl }) + assert.equal(raised.code, 'memory_diverged') + assert.equal(raised.rollback_succeeded, true) + assert.equal(rollback, true) assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-04.json'), 'utf8')).machine.memory, '512m') } finally { fs.rmSync(root, { recursive: true, force: true }) @@ -152,9 +170,15 @@ test('raiseDockerSlotMemory refuses a mismatched cgroup limit without persisting tmpVmRecord(root, 'vm-05') const vm = { id: 'vm-05', runtime_type: 'docker', machine: { memory: '512m' } } let updated = false + let rollback = false const execImpl = (argv) => { if (argv.includes('update')) { - updated = true + if (argv.includes('512m') && updated) { + rollback = true + updated = false + } else { + updated = true + } return { ok: true, stdout: '' } } if (argv.includes('inspect')) { @@ -164,9 +188,66 @@ test('raiseDockerSlotMemory refuses a mismatched cgroup limit without persisting if (argv.includes('exec')) return { ok: true, stdout: String(512 * 1024 ** 2) } return { ok: false, stderr: 'unexpected' } } - assert.equal(raiseDockerSlotMemory(root, vm, '1g', { execImpl }).code, 'cgroup_memory_diverged') + const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl }) + assert.equal(raised.code, 'cgroup_memory_diverged') + assert.equal(raised.rollback_succeeded, true) + assert.equal(rollback, true) assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-05.json'), 'utf8')).machine.memory, '512m') } finally { fs.rmSync(root, { recursive: true, force: true }) } }) + +test('raiseDockerSlotMemory accepts cgroup v1 limit when memory.max is missing', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-mem-cgroup-v1-')) + try { + const id = 'vm-07' + tmpVmRecord(root, id) + const vm = { id, runtime_type: 'docker', machine: { memory: '512m' } } + let updated = false + const execImpl = (argv) => { + if (argv.includes('update')) { + updated = true + return { ok: true, stdout: '' } + } + if (argv.includes('inspect')) { + const bytes = updated ? 1024 ** 3 : 512 * 1024 ** 2 + return { ok: true, stdout: `${bytes} ${bytes} true` } + } + if (argv.join(' ').includes('memory.max')) return { ok: false, stderr: 'No such file' } + if (argv.join(' ').includes('limit_in_bytes')) return { ok: true, stdout: String(1024 ** 3) } + return { ok: false, stderr: 'unexpected' } + } + const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl }) + assert.equal(raised.ok, true) + assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'vms', `${id}.json`), 'utf8')).machine.memory, '1g') + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('raiseDockerSlotMemory accepts memory.max=max when HostConfig already matches', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-mem-cgroup-max-')) + try { + const id = 'vm-08' + tmpVmRecord(root, id) + const vm = { id, runtime_type: 'docker', machine: { memory: '512m' } } + let updated = false + const execImpl = (argv) => { + if (argv.includes('update')) { + updated = true + return { ok: true, stdout: '' } + } + if (argv.includes('inspect')) { + const bytes = updated ? 1024 ** 3 : 512 * 1024 ** 2 + return { ok: true, stdout: `${bytes} ${bytes} true` } + } + if (argv.includes('exec')) return { ok: true, stdout: 'max' } + return { ok: false, stderr: 'unexpected' } + } + const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl }) + assert.equal(raised.ok, true) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +})