feat: drain and patch individual CLI Docker pool slots

README.md

@@ -1236,3 +1236,50 @@ Normalized event replay remains separate.
 Native CLI stores remain in their configured private homes and require independent retention with credential files excluded.
 The process manifest does not claim native-store completeness, trusted image identity, or coverage of native subprocess stores.
 Retained native-session API jobs do not yet enter this one-shot capture scope.
+
+### Individually maintained Docker pool slots
+
+The bearer-protected `/admin/pools/status` endpoint reports explicit adopted pool
+identities and durable roll receipts. Maintenance requires `BRIDGE_BEARER`, even
+on loopback. It does not restart other bridge services or stop their jobs.
+
+For an inactive historical pool, `POST /admin/pools/adopt` accepts `prefix`,
+`legacyDataDir`, `legacyPorts`, immutable `expectedImage`, and ordered contiguous
+`slots: [{index, id}]` with full container IDs. The bridge takes the existing
+legacy data-directory ownership lock, requires the old listener ports to be
+unused, and verifies every named slot has no Docker execs and only its idle
+`dumb-init`/`tail` processes. Each slot must have a bind associated with that
+legacy directory. Refused adoption leaves the containers untouched. Adoption
+starts every slot drained; it never provisions, removes, or resets credentials.
+
+`POST /admin/pools/:prefix/slots/:index/drain` takes `expectedId` and prevents new
+leases for that slot. Existing leases finish normally; other slots remain
+available. `POST .../roll` takes the same `expectedId` and immutable local
+`imageId` (`sha256:...`); an occupied slot returns 409. An idle candidate retains
+all existing named/anonymous volumes, binds, process configuration, operator
+environment overrides, and resource settings. Unsupported namespace/device/port
+layouts refuse before replacement. The candidate must pass the maintained Node
+version policy in `docker/Dockerfile.cli-runtime`, per-start setup, and normal
+Docker preflight before the old container stops. The old container remains
+stopped under a retained name, with its data and image available for inspection.
+No image is pulled and no volume is deleted by this interface.
+
+`pool-maintenance.json` under the bridge data directory records exact old and
+candidate identities before each Docker transition. After an interrupted roll,
+retry the same original ID and image; a different intent is refused. After owner
+restart, adopted slots remain drained until the same roll request verifies and
+re-admits each completed candidate. The journal contains no credential values.
+Use the registered service deployer to activate merged source; an unchanged
+lockfile and matching installed lock marker allow its existing source-only path.
+
+Run the affected real Docker/HTTP proof on an admitted build host with an
+already-present immutable runtime image:
+
+```sh
+CLI_ROLL_IMAGE=sha256:<existing-image-id> pnpm exec tsx scripts/verify-pool-runtime-roll.ts
+```
+
+This fixture proves ownership, drain, configuration preservation and restart
+reconciliation using local containers; it makes no model requests. Production
+completion additionally requires the actual eight slot IDs to report the patched
+Node version and their active immutable image's retained vulnerability scan.

scripts/verify-pool-runtime-roll.ts

@@ -0,0 +1,160 @@
+#!/usr/bin/env tsx
+/** Actual Docker + authenticated HTTP proof; no provider requests or customer data. */
+import assert from 'node:assert/strict'
+import { randomBytes } from 'node:crypto'
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { serve } from '@hono/node-server'
+import { ContainerPool } from '../src/executors/container-pool.js'
+import { dockerCli } from '../src/executors/docker-cli.js'
+import { dockerEngine, replaceRuntime, type RollReceipt } from '../src/executors/docker-runtime-roll.js'
+import { acquireDataDirectoryLock } from '../src/runtime/single-instance.js'
+import { loadConfig } from '../src/config.js'
+import { buildApp } from '../src/server.js'
+
+const image = process.env.CLI_ROLL_IMAGE
+if (!image || !/^sha256:[a-f0-9]{64}$/u.test(image)) throw new Error('CLI_ROLL_IMAGE immutable existing local image required')
+const token = randomBytes(24).toString('hex')
+const suffix = randomBytes(5).toString('hex')
+const prefix = `cli-bridge-proof-${suffix}`
+const root = mkdtempSync(join(tmpdir(), `${prefix}-`))
+const legacy = join(root, 'legacy', 'data')
+const ws = join(root, 'legacy', 'ws')
+const data = join(root, 'current')
+for (const dir of [legacy, ws, data]) mkdirSync(dir, { recursive: true, mode: 0o700 })
+const owned = new Set<string>()
+const volumes = new Set<string>()
+const checks: string[] = []
+let oldImage = ''
+let pool: ContainerPool | undefined
+let appOwner: Awaited<ReturnType<typeof buildApp>> | undefined
+let server: ReturnType<typeof serve> | undefined
+let base = ''
+
+async function command(args: string[]): Promise<string> {
+  const result = await dockerCli(args, { timeoutMs: 120_000 })
+  if (result.code) throw new Error(`fixture Docker command failed: ${args[0]}`)
+  return result.stdout.trim()
+}
+async function api(path: string, body?: unknown, auth = true) {
+  return await fetch(`${base}${path}`, { method: body === undefined ? 'GET' : 'POST',
+    headers: { ...(auth ? { authorization: `Bearer ${token}` } : {}), 'content-type': 'application/json' },
+    ...(body === undefined ? {} : { body: JSON.stringify(body) }) })
+}
+async function startApp() {
+  appOwner = await buildApp(loadConfig({ BRIDGE_BACKENDS: 'codex', CODEX_EXECUTOR: 'host',
+    BRIDGE_DATA_DIR: data, BRIDGE_BEARER: token, BRIDGE_HOST: '127.0.0.1' }))
+  server = serve({ fetch: appOwner.app.fetch, hostname: '127.0.0.1', port: 0 })
+  await new Promise<void>((resolve) => server!.listening ? resolve() : server!.once('listening', resolve))
+  base = `http://127.0.0.1:${(server.address() as { port: number }).port}`
+}
+async function stopApp() {
+  if (server) await new Promise<void>((resolve, reject) => server!.close((e) => e ? reject(e) : resolve()))
+  if (appOwner) {
+    for (const hook of appOwner.extras.shutdownHooks) await hook()
+    appOwner.sessions.close()
+  }
+  server = undefined; appOwner = undefined
+}
+
+try {
+  // Distinct immutable old image, sharing the existing runtime's layers. This
+  // fixture proves replacement mechanics; actual Node22->24 belongs to live proof.
+  const seed = await command(['create', '--name', `${prefix}-seed`, image])
+  owned.add(seed)
+  oldImage = await command(['commit', '--change', `LABEL cli-bridge.proof=${suffix}`, seed])
+  await command(['rm', '-v', seed]); owned.delete(seed)
+
+  pool = await ContainerPool.create({ size: 2, image: oldImage, namePrefix: `${prefix}-leases`,
+    oauthMode: 'share', workspaceRoot: ws, slotMaxHoldMs: 0 })
+  for (const id of pool.liveContainerIds()) owned.add(id)
+  const first = await pool.acquire()
+  const worker = dockerCli(['exec', first.containerId, 'sleep', '2'])
+  pool.drainSlot(first.slotIndex, first.containerId)
+  let replacedBusy = false
+  await assert.rejects(pool.rollSlot(first.slotIndex, first.containerId, image, async () => { replacedBusy = true; return '' }))
+  assert.equal(replacedBusy, false)
+  const other = await pool.acquire()
+  assert.notEqual(other.slotIndex, first.slotIndex)
+  other.release()
+  assert.equal((await worker).code, 0)
+  first.release()
+  const next = await pool.acquire()
+  assert.notEqual(next.slotIndex, first.slotIndex)
+  next.release()
+  checks.push('drain preserves real active exec and routes other work to another slot')
+  let armed = false
+  const receipt: RollReceipt = { oldId: first.containerId, imageId: image,
+    name: `${prefix}-leases-${first.slotIndex}`, nonce: randomBytes(16).toString('hex'), phase: 'intent' }
+  const replacement = await pool.rollSlot(first.slotIndex, first.containerId, image, async (_id, arm) =>
+    await replaceRuntime(receipt, () => {}, async (id) => { await arm(id); armed = true }))
+  owned.add(replacement)
+  assert.equal(armed, true)
+  assert.equal((await dockerEngine('GET', `/containers/${first.containerId}/json`)).State.Running, false)
+  checks.push('drained idle slot promotes immutable image and retains stopped old container')
+  await pool.destroy(); pool = undefined
+
+  const slots = []
+  for (let index = 0; index < 2; index++) {
+    const volume = `${prefix}-auth-${index}`
+    volumes.add(volume)
+    const id = await command(['run', '-d', '--name', `${prefix}-${index}`, '--restart', 'on-failure:3',
+      '--memory', '128m', '--cpus', '1', '--env', `PROOF_SECRET=${token}`, '--mount', `type=bind,source=${ws},target=${ws}`,
+      '--mount', `type=volume,source=${volume},target=/root/.config/opencode`, oldImage, 'tail', '-f', '/dev/null'])
+    owned.add(id); slots.push({ index, id })
+    await command(['exec', id, 'sh', '-c', 'printf retained > /root/.config/opencode/proof-marker'])
+  }
+  await startApp()
+  assert.equal((await api('/admin/pools/status', undefined, false)).status, 401)
+  const manifest = { prefix, legacyDataDir: legacy, legacyPorts: [49171], expectedImage: oldImage, slots }
+  // A real unexpected process blocks adoption; the failed attempt releases its
+  // otherwise-unreferenced data-directory lock.
+  const unexpected = dockerCli(['exec', slots[0]!.id, 'sleep', '3'])
+  await new Promise((resolve) => setTimeout(resolve, 200))
+  assert.equal((await api('/admin/pools/adopt', manifest)).status, 409)
+  const lock = acquireDataDirectoryLock(legacy); lock.release()
+  assert.equal((await unexpected).code, 0)
+  assert.equal((await api('/admin/pools/adopt', manifest)).status, 200)
+  assert.equal((await api('/admin/pools/adopt', manifest)).status, 409)
+  checks.push('authenticated adoption refuses unexpected work and releases refused ownership lock')
+
+  for (const slot of slots) {
+    assert.equal((await api(`/admin/pools/${prefix}/slots/${slot.index}/drain`, { expectedId: slot.id })).status, 200)
+    const response = await api(`/admin/pools/${prefix}/slots/${slot.index}/roll`, { expectedId: slot.id, imageId: image })
+    if (response.status !== 200) throw new Error(`actual HTTP roll refused: ${JSON.stringify(await response.json())}`)
+    const body = await response.json() as { receipt: RollReceipt }
+    const id = body.receipt.candidateId!
+    owned.add(id)
+    const info = await dockerEngine('GET', `/containers/${id}/json`)
+    assert.equal(info.Image, image)
+    assert.ok(info.Config.Env.includes(`PROOF_SECRET=${token}`))
+    assert.equal(info.HostConfig.Memory, 128 * 1024 * 1024)
+    assert.equal(await command(['exec', id, 'cat', '/root/.config/opencode/proof-marker']), 'retained')
+    assert.ok(info.Mounts.some((m: any) => m.Source === ws && m.Destination === ws && m.RW))
+    assert.equal((await dockerEngine('GET', `/containers/${slot.id}/json`)).State.Running, false)
+  }
+  checks.push('two real HTTP rolls preserve credential volumes, bind mounts, private env and resources')
+  await stopApp()
+  await startApp()
+  for (const slot of slots) {
+    assert.equal((await api(`/admin/pools/${prefix}/slots/${slot.index}/roll`, { expectedId: slot.id, imageId: image })).status, 200)
+  }
+  const status = await (await api('/admin/pools/status')).json() as { pools: Array<{ slots: Array<{ draining: boolean }> }> }
+  assert.ok(status.pools[0]!.slots.every((slot) => !slot.draining))
+  assert.equal(readFileSync(join(data, 'pool-maintenance.json'), 'utf8').includes(token), false)
+  checks.push('actual HTTP owner restart reconciles completed receipts and re-admits verified slots without secrets in journal')
+  const result = { complete: true, checks, old_fixture_image: oldImage, candidate_image: image,
+    production_node_upgrade_proven: false, no_model_calls: true }
+  const output = process.env.CLI_ROLL_PROOF_OUTPUT
+  if (output) writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 })
+  console.log(JSON.stringify(result))
+} finally {
+  await stopApp()
+  if (pool) await pool.destroy()
+  // All names/IDs below were created by this invocation. No shared pruning.
+  for (const id of owned) await dockerCli(['rm', '-f', '-v', id])
+  for (const volume of volumes) await dockerCli(['volume', 'rm', volume])
+  if (oldImage) await dockerCli(['image', 'rm', oldImage])
+  rmSync(root, { recursive: true, force: true })
+}