fix(vm): Docker 在线扩容 cgroup 校验与失败回滚

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 未发布
+
+- Docker 槽在线扩容：面板成功须 HostConfig 内存与 swap 一致；可读 cgroup 时核对 v2 `memory.max` 或 v1 `memory.limit_in_bytes`（`max`/无上限哨兵在 HostConfig 已匹配时跳过）。扩容后校验失败会 docker update 回滚至扩容前限额且不写 vm 配置。（#623，REQ-0441）
+
 ## 1.3.201 — 2026-10-20
 
 - 消息线程 beta 在控制面按消息 id 记住 24 小时（最多 1 万条）：同模型的 `continue` 补回上一轮 beta，并保留本轮新算出的；换模型、过期或未知 id 用现算值，不失败。create 轮对支持的模型仅凭 `thread` 带上 `mid-conversation-output-config`，continue 不再回放 effort 控件也能带上该门。命中 / 未命中 / 换模型记在 `hop_meta.thread_betas`。进程重启后 pin 丢失，按未命中处理。（#621）

docs/DEPLOY.md

@@ -28,7 +28,7 @@ VM2API_DB_SECRET='再一串'
 
 不是一个父容器里多个子进程。
 
-槽位默认内存上限为 `512m`，在面板「规格」或 `routing.json` 的 `vm.memory` 配置（`256m` / `512m` / `1g` / `2g` / `4g` / `8g` / `16g`）。创建时可按槽覆盖，结果写在该槽 `vm.machine.memory`。已不再读取 `KIN_VM_MEMORY`。低于 1G 时，常驻 native host 与临时官方 CLI 同时运行可能 OOM；额度 API 探测本身不应额外启动推理 CLI。已有容器不会因控制面升级自动改内存，可在核对宿主余量后使用面板在线扩容，不重建容器。面板成功响应以 Docker 的内存和 swap 上限、容器内 `memory.max` 三者均匹配为准；内核每次准入按当前 `memory.max` 动态计算阈值，并按请求实际序列化大小限制在途负载。单次请求的 CLI 内存突增仍可能超过 512 MiB，准入不保证任意请求都能在此上限内运行。
+槽位默认内存上限为 `512m`，在面板「规格」或 `routing.json` 的 `vm.memory` 配置（`256m` / `512m` / `1g` / `2g` / `4g` / `8g` / `16g`）。创建时可按槽覆盖，结果写在该槽 `vm.machine.memory`。已不再读取 `KIN_VM_MEMORY`。低于 1G 时，常驻 native host 与临时官方 CLI 同时运行可能 OOM；额度 API 探测本身不应额外启动推理 CLI。已有容器不会因控制面升级自动改内存，可在核对宿主余量后使用面板在线扩容，不重建容器。面板成功须 Docker HostConfig 的 memory 与 memory-swap 均等于请求值；若容器内可读 cgroup 限额（cgroup v2 的 `memory.max`，否则 v1 的 `memory/memory.limit_in_bytes`），则须与请求一致；读不到路径或值为 `max`/无上限哨兵时，在 HostConfig 已匹配的前提下跳过 cgroup 核对。`docker update` 成功后若后续校验失败，控制面会用扩容前的 memory/swap **回滚**容器限额且**不写** vm 配置；回滚失败时 API 会提示手动核对。内核每次准入按当前 cgroup 限额动态计算阈值，并按请求实际序列化大小限制在途负载。单次请求的 CLI 内存突增仍可能超过 512 MiB，准入不保证任意请求都能在此上限内运行。
 
 KVM 槽需要宿主 `/dev/kvm` 可读写（控制面在容器内时，用一次 `--device /dev/kvm` 的探测容器确认）。runner 降权给槽位用户跑 QEMU 时会带上 `/dev/kvm` 的设备组，属组与槽位主 GID 不同也能用。已有 KVM 槽要更新 `kin-kvm-*` 镜像并重建一次。没有 KVM 时不要在生产打开 `vm.allow_tcg`：那是 TCG 软件模拟，仅供测试。
 

src/lib/vm/docker-slot-memory.mjs

@@ -55,6 +55,68 @@ function dockerExec(vm, argv, execImpl, timeout) {
   return execImpl(argv, { timeout, ...(env ? { env } : {}) })
 }
 
+/** cgroup v1「无上限」哨兵（与 Linux PAGE_COUNTER_MAX 同量级） */
+const CGROUP_V1_UNLIMITED = 9223372036854771712
+
+export function parseCgroupMemoryLimit(raw) {
+  const text = String(raw ?? '').trim()
+  if (!text) return { kind: 'unreadable' }
+  if (text.toLowerCase() === 'max') return { kind: 'unlimited' }
+  if (/^\d+$/.test(text)) {
+    try {
+      const big = BigInt(text)
+      if (big >= BigInt(CGROUP_V1_UNLIMITED) - 1024n) return { kind: 'unlimited' }
+      const bytes = Number(big)
+      if (!Number.isSafeInteger(bytes) || bytes < 0) return { kind: 'unreadable' }
+      return { kind: 'limited', bytes }
+    } catch {
+      return { kind: 'unreadable' }
+    }
+  }
+  return { kind: 'unreadable' }
+}
+
+export function readContainerCgroupMemoryLimit(vm, containerId, execImpl) {
+  const v2 = dockerExec(vm, ['docker', 'exec', containerId, 'cat', '/sys/fs/cgroup/memory.max'], execImpl, 10_000)
+  if (v2.ok) {
+    const parsed = parseCgroupMemoryLimit(v2.stdout)
+    if (parsed.kind !== 'unreadable') return parsed
+  }
+  const v1 = dockerExec(
+    vm,
+    ['docker', 'exec', containerId, 'cat', '/sys/fs/cgroup/memory/memory.limit_in_bytes'],
+    execImpl,
+    10_000,
+  )
+  if (v1.ok) {
+    const parsed = parseCgroupMemoryLimit(v1.stdout)
+    if (parsed.kind !== 'unreadable') return parsed
+  }
+  return { kind: 'unreadable' }
+}
+
+function rollbackDockerMemory(vm, before, execImpl) {
+  const mem = before.memory
+  const swap =
+    before.swapBytes > 0 && before.swapBytes !== before.bytes ? formatDockerMemoryBytes(before.swapBytes) || mem : mem
+  const name = containerName(vm.id)
+  const r = dockerExec(vm, ['docker', 'update', '--memory', mem, '--memory-swap', swap, name], execImpl, 15_000)
+  return r.ok
+}
+
+function failAfterDockerUpdate(vm, before, execImpl, failure) {
+  const rollbackOk = rollbackDockerMemory(vm, before, execImpl)
+  const rollbackNote = rollbackOk
+    ? '已用 docker update 回滚至扩容前限额，vm 配置未写入。'
+    : '自动回滚 docker update 失败，请手动核对容器 HostConfig 与面板配置是否一致。'
+  return {
+    ...failure,
+    message: `${failure.message} ${rollbackNote}`.trim(),
+    config_unchanged: true,
+    rollback_succeeded: rollbackOk,
+  }
+}
+
 export function inspectDockerContainerMemory(vm, execImpl = defaultExec) {
   const name = containerName(vm?.id)
   const r = dockerExec(
@@ -140,32 +202,32 @@ export function raiseDockerSlotMemory(projectRoot, vm, requestedMemory, { routin
 
   const inspected = inspectDockerContainerMemory(vm, execImpl)
   if (!inspected.ok) {
-    return {
+    return failAfterDockerUpdate(vm, before, execImpl, {
       ok: false,
       code: inspected.code,
       message: inspected.message,
       requested: mem,
-    }
+    })
   }
   const requestedBytes = memoryMiB(mem) * 1024 ** 2
   if (inspected.bytes !== requestedBytes || inspected.swapBytes !== requestedBytes) {
-    return {
+    return failAfterDockerUpdate(vm, before, execImpl, {
       ok: false,
       code: 'memory_diverged',
       message: `容器内存或 swap 上限与请求的 ${mem} 不一致，未写入配置`,
       requested: mem,
       container_memory: inspected.memory,
-    }
+    })
   }
-  const cgroup = dockerExec(vm, ['docker', 'exec', name, 'cat', '/sys/fs/cgroup/memory.max'], execImpl, 10_000)
-  if (!cgroup.ok || Number(String(cgroup.stdout || '').trim()) !== requestedBytes) {
-    return {
+  const cgroup = readContainerCgroupMemoryLimit(vm, name, execImpl)
+  if (cgroup.kind === 'limited' && cgroup.bytes !== requestedBytes) {
+    return failAfterDockerUpdate(vm, before, execImpl, {
       ok: false,
       code: 'cgroup_memory_diverged',
       message: '容器实际 cgroup 内存上限与请求不一致，未写入配置',
       requested: mem,
       container_memory: inspected.memory,
-    }
+    })
   }
 
   const saved = patchVmRecord(projectRoot, vm.id, (record) => {
@@ -178,13 +240,13 @@ export function raiseDockerSlotMemory(projectRoot, vm, requestedMemory, { routin
     return next
   })
   if (!saved) {
-    return {
+    return failAfterDockerUpdate(vm, before, execImpl, {
       ok: false,
       code: 'vm_not_found',
       message: 'vm not found',
       requested: mem,
       container_memory: inspected.memory,
-    }
+    })
   }
 
   return {

test/unit/docker-slot-memory.test.mjs

@@ -6,6 +6,7 @@ import path from 'node:path'
 import {
   formatDockerMemoryBytes,
   inspectDockerContainerMemory,
+  parseCgroupMemoryLimit,
   raiseDockerSlotMemory,
   validateDockerMemoryRaise,
 } from '../../src/lib/vm/docker-slot-memory.mjs'
@@ -15,6 +16,13 @@ test('formatDockerMemoryBytes normalizes docker byte limits', () => {
   assert.equal(formatDockerMemoryBytes(1073741824), '1g')
 })
 
+test('parseCgroupMemoryLimit handles v2 max and v1 unlimited sentinel', () => {
+  assert.deepEqual(parseCgroupMemoryLimit('max'), { kind: 'unlimited' })
+  assert.deepEqual(parseCgroupMemoryLimit('9223372036854771712'), { kind: 'unlimited' })
+  assert.deepEqual(parseCgroupMemoryLimit(String(1024 ** 3)), { kind: 'limited', bytes: 1024 ** 3 })
+  assert.deepEqual(parseCgroupMemoryLimit(''), { kind: 'unreadable' })
+})
+
 test('validateDockerMemoryRaise rejects kvm and invalid or non-increasing memory', () => {
   const kvm = { id: 'vm-1', runtime_type: 'kvm', machine: { memory: '512m' } }
   const kvmCheck = validateDockerMemoryRaise(kvm, '1g', {})
@@ -128,18 +136,28 @@ test('raiseDockerSlotMemory refuses a mismatched swap limit without persisting',
     tmpVmRecord(root, 'vm-04')
     const vm = { id: 'vm-04', runtime_type: 'docker', machine: { memory: '512m' } }
     let updated = false
+    let rollback = false
     const execImpl = (argv) => {
       if (argv.includes('update')) {
-        updated = true
+        if (argv.includes('512m') && updated) {
+          rollback = true
+          updated = false
+        } else {
+          updated = true
+        }
         return { ok: true, stdout: '' }
       }
       if (argv.includes('inspect')) {
         const memory = updated ? 1024 ** 3 : 512 * 1024 ** 2
-        return { ok: true, stdout: `${memory} ${512 * 1024 ** 2} true` }
+        const swap = updated ? 512 * 1024 ** 2 : 512 * 1024 ** 2
+        return { ok: true, stdout: `${memory} ${swap} true` }
       }
       return { ok: false, stderr: 'unexpected' }
     }
-    assert.equal(raiseDockerSlotMemory(root, vm, '1g', { execImpl }).code, 'memory_diverged')
+    const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl })
+    assert.equal(raised.code, 'memory_diverged')
+    assert.equal(raised.rollback_succeeded, true)
+    assert.equal(rollback, true)
     assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-04.json'), 'utf8')).machine.memory, '512m')
   } finally {
     fs.rmSync(root, { recursive: true, force: true })
@@ -152,9 +170,15 @@ test('raiseDockerSlotMemory refuses a mismatched cgroup limit without persisting
     tmpVmRecord(root, 'vm-05')
     const vm = { id: 'vm-05', runtime_type: 'docker', machine: { memory: '512m' } }
     let updated = false
+    let rollback = false
     const execImpl = (argv) => {
       if (argv.includes('update')) {
-        updated = true
+        if (argv.includes('512m') && updated) {
+          rollback = true
+          updated = false
+        } else {
+          updated = true
+        }
         return { ok: true, stdout: '' }
       }
       if (argv.includes('inspect')) {
@@ -164,9 +188,66 @@ test('raiseDockerSlotMemory refuses a mismatched cgroup limit without persisting
       if (argv.includes('exec')) return { ok: true, stdout: String(512 * 1024 ** 2) }
       return { ok: false, stderr: 'unexpected' }
     }
-    assert.equal(raiseDockerSlotMemory(root, vm, '1g', { execImpl }).code, 'cgroup_memory_diverged')
+    const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl })
+    assert.equal(raised.code, 'cgroup_memory_diverged')
+    assert.equal(raised.rollback_succeeded, true)
+    assert.equal(rollback, true)
     assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-05.json'), 'utf8')).machine.memory, '512m')
   } finally {
     fs.rmSync(root, { recursive: true, force: true })
   }
 })
+
+test('raiseDockerSlotMemory accepts cgroup v1 limit when memory.max is missing', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-mem-cgroup-v1-'))
+  try {
+    const id = 'vm-07'
+    tmpVmRecord(root, id)
+    const vm = { id, runtime_type: 'docker', machine: { memory: '512m' } }
+    let updated = false
+    const execImpl = (argv) => {
+      if (argv.includes('update')) {
+        updated = true
+        return { ok: true, stdout: '' }
+      }
+      if (argv.includes('inspect')) {
+        const bytes = updated ? 1024 ** 3 : 512 * 1024 ** 2
+        return { ok: true, stdout: `${bytes} ${bytes} true` }
+      }
+      if (argv.join(' ').includes('memory.max')) return { ok: false, stderr: 'No such file' }
+      if (argv.join(' ').includes('limit_in_bytes')) return { ok: true, stdout: String(1024 ** 3) }
+      return { ok: false, stderr: 'unexpected' }
+    }
+    const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl })
+    assert.equal(raised.ok, true)
+    assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'vms', `${id}.json`), 'utf8')).machine.memory, '1g')
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true })
+  }
+})
+
+test('raiseDockerSlotMemory accepts memory.max=max when HostConfig already matches', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-mem-cgroup-max-'))
+  try {
+    const id = 'vm-08'
+    tmpVmRecord(root, id)
+    const vm = { id, runtime_type: 'docker', machine: { memory: '512m' } }
+    let updated = false
+    const execImpl = (argv) => {
+      if (argv.includes('update')) {
+        updated = true
+        return { ok: true, stdout: '' }
+      }
+      if (argv.includes('inspect')) {
+        const bytes = updated ? 1024 ** 3 : 512 * 1024 ** 2
+        return { ok: true, stdout: `${bytes} ${bytes} true` }
+      }
+      if (argv.includes('exec')) return { ok: true, stdout: 'max' }
+      return { ok: false, stderr: 'unexpected' }
+    }
+    const raised = raiseDockerSlotMemory(root, vm, '1g', { execImpl })
+    assert.equal(raised.ok, true)
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true })
+  }
+})