diff --git a/README.md b/README.md index ccc523d..ad8e7e4 100644 --- a/README.md +++ b/README.md @@ -1236,3 +1236,50 @@ Normalized event replay remains separate. Native CLI stores remain in their configured private homes and require independent retention with credential files excluded. The process manifest does not claim native-store completeness, trusted image identity, or coverage of native subprocess stores. Retained native-session API jobs do not yet enter this one-shot capture scope. + +### Individually maintained Docker pool slots + +The bearer-protected `/admin/pools/status` endpoint reports explicit adopted pool +identities and durable roll receipts. Maintenance requires `BRIDGE_BEARER`, even +on loopback. It does not restart other bridge services or stop their jobs. + +For an inactive historical pool, `POST /admin/pools/adopt` accepts `prefix`, +`legacyDataDir`, `legacyPorts`, immutable `expectedImage`, and ordered contiguous +`slots: [{index, id}]` with full container IDs. The bridge takes the existing +legacy data-directory ownership lock, requires the old listener ports to be +unused, and verifies every named slot has no Docker execs and only its idle +`dumb-init`/`tail` processes. Each slot must have a bind associated with that +legacy directory. Refused adoption leaves the containers untouched. Adoption +starts every slot drained; it never provisions, removes, or resets credentials. + +`POST /admin/pools/:prefix/slots/:index/drain` takes `expectedId` and prevents new +leases for that slot. Existing leases finish normally; other slots remain +available. `POST .../roll` takes the same `expectedId` and immutable local +`imageId` (`sha256:...`); an occupied slot returns 409. An idle candidate retains +all existing named/anonymous volumes, binds, process configuration, operator +environment overrides, and resource settings. Unsupported namespace/device/port +layouts refuse before replacement. The candidate must pass the maintained Node +version policy in `docker/Dockerfile.cli-runtime`, per-start setup, and normal +Docker preflight before the old container stops. The old container remains +stopped under a retained name, with its data and image available for inspection. +No image is pulled and no volume is deleted by this interface. + +`pool-maintenance.json` under the bridge data directory records exact old and +candidate identities before each Docker transition. After an interrupted roll, +retry the same original ID and image; a different intent is refused. After owner +restart, adopted slots remain drained until the same roll request verifies and +re-admits each completed candidate. The journal contains no credential values. +Use the registered service deployer to activate merged source; an unchanged +lockfile and matching installed lock marker allow its existing source-only path. + +Run the affected real Docker/HTTP proof on an admitted build host with an +already-present immutable runtime image: + +```sh +CLI_ROLL_IMAGE=sha256: pnpm exec tsx scripts/verify-pool-runtime-roll.ts +``` + +This fixture proves ownership, drain, configuration preservation and restart +reconciliation using local containers; it makes no model requests. Production +completion additionally requires the actual eight slot IDs to report the patched +Node version and their active immutable image's retained vulnerability scan. diff --git a/scripts/verify-pool-runtime-roll.ts b/scripts/verify-pool-runtime-roll.ts new file mode 100644 index 0000000..3913291 --- /dev/null +++ b/scripts/verify-pool-runtime-roll.ts @@ -0,0 +1,160 @@ +#!/usr/bin/env tsx +/** Actual Docker + authenticated HTTP proof; no provider requests or customer data. */ +import assert from 'node:assert/strict' +import { randomBytes } from 'node:crypto' +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { serve } from '@hono/node-server' +import { ContainerPool } from '../src/executors/container-pool.js' +import { dockerCli } from '../src/executors/docker-cli.js' +import { dockerEngine, replaceRuntime, type RollReceipt } from '../src/executors/docker-runtime-roll.js' +import { acquireDataDirectoryLock } from '../src/runtime/single-instance.js' +import { loadConfig } from '../src/config.js' +import { buildApp } from '../src/server.js' + +const image = process.env.CLI_ROLL_IMAGE +if (!image || !/^sha256:[a-f0-9]{64}$/u.test(image)) throw new Error('CLI_ROLL_IMAGE immutable existing local image required') +const token = randomBytes(24).toString('hex') +const suffix = randomBytes(5).toString('hex') +const prefix = `cli-bridge-proof-${suffix}` +const root = mkdtempSync(join(tmpdir(), `${prefix}-`)) +const legacy = join(root, 'legacy', 'data') +const ws = join(root, 'legacy', 'ws') +const data = join(root, 'current') +for (const dir of [legacy, ws, data]) mkdirSync(dir, { recursive: true, mode: 0o700 }) +const owned = new Set() +const volumes = new Set() +const checks: string[] = [] +let oldImage = '' +let pool: ContainerPool | undefined +let appOwner: Awaited> | undefined +let server: ReturnType | undefined +let base = '' + +async function command(args: string[]): Promise { + const result = await dockerCli(args, { timeoutMs: 120_000 }) + if (result.code) throw new Error(`fixture Docker command failed: ${args[0]}`) + return result.stdout.trim() +} +async function api(path: string, body?: unknown, auth = true) { + return await fetch(`${base}${path}`, { method: body === undefined ? 'GET' : 'POST', + headers: { ...(auth ? { authorization: `Bearer ${token}` } : {}), 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) }) +} +async function startApp() { + appOwner = await buildApp(loadConfig({ BRIDGE_BACKENDS: 'codex', CODEX_EXECUTOR: 'host', + BRIDGE_DATA_DIR: data, BRIDGE_BEARER: token, BRIDGE_HOST: '127.0.0.1' })) + server = serve({ fetch: appOwner.app.fetch, hostname: '127.0.0.1', port: 0 }) + await new Promise((resolve) => server!.listening ? resolve() : server!.once('listening', resolve)) + base = `http://127.0.0.1:${(server.address() as { port: number }).port}` +} +async function stopApp() { + if (server) await new Promise((resolve, reject) => server!.close((e) => e ? reject(e) : resolve())) + if (appOwner) { + for (const hook of appOwner.extras.shutdownHooks) await hook() + appOwner.sessions.close() + } + server = undefined; appOwner = undefined +} + +try { + // Distinct immutable old image, sharing the existing runtime's layers. This + // fixture proves replacement mechanics; actual Node22->24 belongs to live proof. + const seed = await command(['create', '--name', `${prefix}-seed`, image]) + owned.add(seed) + oldImage = await command(['commit', '--change', `LABEL cli-bridge.proof=${suffix}`, seed]) + await command(['rm', '-v', seed]); owned.delete(seed) + + pool = await ContainerPool.create({ size: 2, image: oldImage, namePrefix: `${prefix}-leases`, + oauthMode: 'share', workspaceRoot: ws, slotMaxHoldMs: 0 }) + for (const id of pool.liveContainerIds()) owned.add(id) + const first = await pool.acquire() + const worker = dockerCli(['exec', first.containerId, 'sleep', '2']) + pool.drainSlot(first.slotIndex, first.containerId) + let replacedBusy = false + await assert.rejects(pool.rollSlot(first.slotIndex, first.containerId, image, async () => { replacedBusy = true; return '' })) + assert.equal(replacedBusy, false) + const other = await pool.acquire() + assert.notEqual(other.slotIndex, first.slotIndex) + other.release() + assert.equal((await worker).code, 0) + first.release() + const next = await pool.acquire() + assert.notEqual(next.slotIndex, first.slotIndex) + next.release() + checks.push('drain preserves real active exec and routes other work to another slot') + let armed = false + const receipt: RollReceipt = { oldId: first.containerId, imageId: image, + name: `${prefix}-leases-${first.slotIndex}`, nonce: randomBytes(16).toString('hex'), phase: 'intent' } + const replacement = await pool.rollSlot(first.slotIndex, first.containerId, image, async (_id, arm) => + await replaceRuntime(receipt, () => {}, async (id) => { await arm(id); armed = true })) + owned.add(replacement) + assert.equal(armed, true) + assert.equal((await dockerEngine('GET', `/containers/${first.containerId}/json`)).State.Running, false) + checks.push('drained idle slot promotes immutable image and retains stopped old container') + await pool.destroy(); pool = undefined + + const slots = [] + for (let index = 0; index < 2; index++) { + const volume = `${prefix}-auth-${index}` + volumes.add(volume) + const id = await command(['run', '-d', '--name', `${prefix}-${index}`, '--restart', 'on-failure:3', + '--memory', '128m', '--cpus', '1', '--env', `PROOF_SECRET=${token}`, '--mount', `type=bind,source=${ws},target=${ws}`, + '--mount', `type=volume,source=${volume},target=/root/.config/opencode`, oldImage, 'tail', '-f', '/dev/null']) + owned.add(id); slots.push({ index, id }) + await command(['exec', id, 'sh', '-c', 'printf retained > /root/.config/opencode/proof-marker']) + } + await startApp() + assert.equal((await api('/admin/pools/status', undefined, false)).status, 401) + const manifest = { prefix, legacyDataDir: legacy, legacyPorts: [49171], expectedImage: oldImage, slots } + // A real unexpected process blocks adoption; the failed attempt releases its + // otherwise-unreferenced data-directory lock. + const unexpected = dockerCli(['exec', slots[0]!.id, 'sleep', '3']) + await new Promise((resolve) => setTimeout(resolve, 200)) + assert.equal((await api('/admin/pools/adopt', manifest)).status, 409) + const lock = acquireDataDirectoryLock(legacy); lock.release() + assert.equal((await unexpected).code, 0) + assert.equal((await api('/admin/pools/adopt', manifest)).status, 200) + assert.equal((await api('/admin/pools/adopt', manifest)).status, 409) + checks.push('authenticated adoption refuses unexpected work and releases refused ownership lock') + + for (const slot of slots) { + assert.equal((await api(`/admin/pools/${prefix}/slots/${slot.index}/drain`, { expectedId: slot.id })).status, 200) + const response = await api(`/admin/pools/${prefix}/slots/${slot.index}/roll`, { expectedId: slot.id, imageId: image }) + if (response.status !== 200) throw new Error(`actual HTTP roll refused: ${JSON.stringify(await response.json())}`) + const body = await response.json() as { receipt: RollReceipt } + const id = body.receipt.candidateId! + owned.add(id) + const info = await dockerEngine('GET', `/containers/${id}/json`) + assert.equal(info.Image, image) + assert.ok(info.Config.Env.includes(`PROOF_SECRET=${token}`)) + assert.equal(info.HostConfig.Memory, 128 * 1024 * 1024) + assert.equal(await command(['exec', id, 'cat', '/root/.config/opencode/proof-marker']), 'retained') + assert.ok(info.Mounts.some((m: any) => m.Source === ws && m.Destination === ws && m.RW)) + assert.equal((await dockerEngine('GET', `/containers/${slot.id}/json`)).State.Running, false) + } + checks.push('two real HTTP rolls preserve credential volumes, bind mounts, private env and resources') + await stopApp() + await startApp() + for (const slot of slots) { + assert.equal((await api(`/admin/pools/${prefix}/slots/${slot.index}/roll`, { expectedId: slot.id, imageId: image })).status, 200) + } + const status = await (await api('/admin/pools/status')).json() as { pools: Array<{ slots: Array<{ draining: boolean }> }> } + assert.ok(status.pools[0]!.slots.every((slot) => !slot.draining)) + assert.equal(readFileSync(join(data, 'pool-maintenance.json'), 'utf8').includes(token), false) + checks.push('actual HTTP owner restart reconciles completed receipts and re-admits verified slots without secrets in journal') + const result = { complete: true, checks, old_fixture_image: oldImage, candidate_image: image, + production_node_upgrade_proven: false, no_model_calls: true } + const output = process.env.CLI_ROLL_PROOF_OUTPUT + if (output) writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 }) + console.log(JSON.stringify(result)) +} finally { + await stopApp() + if (pool) await pool.destroy() + // All names/IDs below were created by this invocation. No shared pruning. + for (const id of owned) await dockerCli(['rm', '-f', '-v', id]) + for (const volume of volumes) await dockerCli(['volume', 'rm', volume]) + if (oldImage) await dockerCli(['image', 'rm', oldImage]) + rmSync(root, { recursive: true, force: true }) +} diff --git a/src/executors/container-pool.ts b/src/executors/container-pool.ts index f0caec3..fd39892 100644 --- a/src/executors/container-pool.ts +++ b/src/executors/container-pool.ts @@ -59,15 +59,14 @@ * the slot that last served that session. */ -import { execFile } from 'node:child_process' import { createHash } from 'node:crypto' -import { promisify } from 'node:util' import { assertDockerNetworkName } from './docker-network.js' -import { containerShell, dockerCli, type DockerCli } from './docker-cli.js' -import { buildCommandFor, isInside } from './docker-preflight.js' +import { dockerCli, type DockerCli } from './docker-cli.js' +import { buildCommandFor } from './docker-preflight.js' import { ExecutorSaturatedError } from './types.js' -const execFileAsync = promisify(execFile) +import { provisionSlot, destroySlot, armContainerStart, START_FORMAT } from './container-slot.js' +export { buildContainerRunArgs } from './container-slot.js' export interface ContainerPoolOptions { /** Number of containers in the pool. */ @@ -171,10 +170,12 @@ export interface AcquiredSlot { release(): void } -interface SlotState { +export interface SlotState { containerId: string index: number busy: boolean + draining?: boolean + maintenance?: boolean /** Slot is dead (provision keeps failing). Not routed to. */ dead: boolean /** Last sessionId served (for sticky routing). */ @@ -225,6 +226,7 @@ export class ContainerPool { private readonly livenessTtlMs: number private readonly cli: DockerCli private destroyed = false + private adopted = false /** Counters for /metrics. */ private counters = { @@ -272,6 +274,49 @@ export class ContainerPool { return new ContainerPool(slots, opts) } + /** Adopt explicit idle identities without provisioning or removing any container. */ + static adopt(opts: ContainerPoolOptions, ids: string[]): ContainerPool { + if (ids.length !== opts.size || !ids.length) throw new Error('invalid adoption size') + const pool = new ContainerPool(ids.map((containerId, index) => ({ containerId, index, + busy: false, draining: true, dead: false, lastSession: null, holdTimer: null, + generation: 0, consecutiveFailures: 0, lastVerifiedAt: Date.now(), armedStart: '' })), opts) + pool.adopted = true + return pool + } + + slotSnapshot() { + return this.slots.map(({ index, containerId, busy, draining, maintenance }) => + ({ index, containerId, busy, draining: Boolean(draining), maintenance: Boolean(maintenance) })) + } + + drainSlot(index: number, expectedId: string): void { + const slot = this.slots[index] + if (!slot || slot.containerId !== expectedId) throw new Error('slot identity changed') + slot.draining = true + } + + /** Exclusive maintenance reservation; never terminates an acquired lease. */ + async rollSlot(index: number, expectedId: string, image: string, replace: (id: string, arm: (id: string) => Promise) => Promise): Promise { + const slot = this.slots[index] + if (!slot || slot.containerId !== expectedId || !slot.draining || slot.busy || slot.maintenance) { + throw new Error('slot must be drained, idle and match expected identity') + } + slot.maintenance = true + try { + const id = await replace(slot.containerId, async (candidate) => { + await armContainerStart(this.opts, candidate, index, this.cli) + }) + this.opts.image = image + slot.containerId = id + slot.generation += 1 + slot.lastVerifiedAt = 0 + slot.armedStart = await armContainerStart(this.opts, id, index, this.cli) + slot.draining = false + this.serveWaiterWith(slot) + return id + } finally { slot.maintenance = false } + } + get size(): number { return this.slots.length } /** @@ -327,15 +372,15 @@ export class ContainerPool { ? this.slots[createHash('sha256').update(sessionId).digest().readUInt32BE(0) % this.slots.length] : undefined if (strictSlot?.dead) throw new Error(`container-pool: retained session slot ${strictSlot.index} is dead`) - if (strictSlot && !strictSlot.busy) return await this.handOut(strictSlot, sessionId, signal) + if (strictSlot && !strictSlot.busy && !strictSlot.draining) return await this.handOut(strictSlot, sessionId, signal) // Sticky preference: prefer a free, non-dead slot that last served // this session. if (!strictSlot && sessionId) { - const sticky = this.slots.find((s) => !s.busy && !s.dead && s.lastSession === sessionId) + const sticky = this.slots.find((s) => !s.busy && !s.dead && !s.draining && s.lastSession === sessionId) if (sticky) return await this.handOut(sticky, sessionId, signal) } - const free = strictSlot ? undefined : this.slots.find((s) => !s.busy && !s.dead) + const free = strictSlot ? undefined : this.slots.find((s) => !s.busy && !s.dead && !s.draining) if (free) return await this.handOut(free, sessionId, signal) // All slots busy or dead — count alive slots so we don't queue @@ -431,7 +476,7 @@ export class ContainerPool { for (const s of this.slots) { if (s.holdTimer) clearTimeout(s.holdTimer) } - await Promise.all(this.slots.map((s) => destroySlot(s.containerId))) + if (!this.adopted) await Promise.all(this.slots.map((s) => destroySlot(s.containerId))) } /** @@ -449,7 +494,7 @@ export class ContainerPool { // Busy slots are healed by their holder's release path or the hold // watchdog; recycling underneath an in-flight exec would swap the // container id out from under it. - if (slot.busy) return + if (slot.busy || slot.draining) return await this.recycleSlot(slot) } @@ -478,6 +523,7 @@ export class ContainerPool { try { await this.ensureSlotUsable(slot) signal?.throwIfAborted() + if (slot.draining) throw new Error('slot drained during acquisition') } catch (err) { slot.busy = false this.serveWaiterWith(slot) @@ -620,13 +666,14 @@ export class ContainerPool { * can take it while it is being verified or rebuilt. */ private serveWaiterWith(slot: SlotState): void { - if (this.destroyed || slot.dead || slot.busy) return + if (this.destroyed || slot.dead || slot.busy || slot.draining) return const waiter = this.takeWaiterFor(slot) if (!waiter) return slot.busy = true void this.ensureSlotUsable(slot).then( () => { slot.busy = false + if (slot.draining) { this.waiters.unshift(waiter); return } // `waiter.resolve` runs markAcquired, which reserves the slot and bumps // the generation the waiter's own release() is bound to. if (waiter.resolve(slot)) { @@ -644,7 +691,7 @@ export class ContainerPool { // This slot could not be made usable. Another alive slot may still // serve the waiter; only when none exists does the waiter learn why. const alternative = waiter.slotIndex === undefined - ? this.slots.find((s) => s !== slot && !s.busy && !s.dead) + ? this.slots.find((s) => s !== slot && !s.busy && !s.dead && !s.draining) : undefined if (alternative) { this.waiters.unshift(waiter) @@ -690,7 +737,7 @@ export class ContainerPool { // waiter's own release() a no-op, leaking the slot as busy until the hold // watchdog fired. if (this.waiters.length > 0) { - const free = this.slots.find((s) => !s.busy && !s.dead) + const free = this.slots.find((s) => !s.busy && !s.dead && !s.draining) if (free) { this.serveWaiterWith(free) } else { @@ -709,6 +756,7 @@ export class ContainerPool { * reserved-slot acquire path) can layer their own ownership rules on top. */ private async reprovisionSlot(slot: SlotState): Promise { + if (this.adopted || slot.draining) return new Error('managed slot requires an explicit runtime roll') if (slot.holdTimer) { clearTimeout(slot.holdTimer); slot.holdTimer = null } this.counters.slot_reprovisions += 1 await destroySlot(slot.containerId) @@ -741,288 +789,3 @@ export class ContainerPool { } } } - -async function provisionSlot( - opts: ContainerPoolOptions, - index: number, - onProgress: (m: string) => void, -): Promise { - const name = `${opts.namePrefix}-${index}` - // Tear down any stale container with the same name (idempotent). - await execFileAsync('docker', ['rm', '-f', name]).catch(() => {}) - - const args = buildContainerRunArgs(opts, index, name) - - onProgress(`[slot ${index}] docker run ${name}`) - const cli = opts.cli ?? dockerCli - const r = await cli(args, { timeoutMs: 120_000 }) - const containerId = r.stdout.trim() - if (r.code !== 0 || !containerId) { - // A missing image is the single most common cause and Docker reports it as - // a registry/pull problem. Name the image and the build command so this - // never again surfaces later as a message about a missing CONTAINER. - const stderr = r.stderr.trim() || r.spawnError || `docker exited ${r.code}` - const missingImage = /No such image|pull access denied|manifest unknown|not found: manifest|Unable to find image/i.test(stderr) - throw new Error( - missingImage - ? `container-pool: cannot create slot ${index} — image ${opts.image} is not available on this host. ` + - `Build it: ${buildCommandFor(opts.image)}. (docker: ${firstLine(stderr)})` - : `container-pool: cannot create slot ${index} — ${firstLine(stderr)}`, - ) - } - // Before anything else touches the container, including this bridge's own - // `docker exec`: the window between `run` and the filter being in place is - // the only moment a slot has unrestricted egress, and the only thing running - // in it is the idle `tail -f /dev/null` entrypoint. - let armedStart = '' - if (opts.afterCreate) { - try { - armedStart = await armContainerStart(opts, containerId, index, cli) - } catch (error) { - await cli(['rm', '-f', containerId], { timeoutMs: 30_000 }) - throw error instanceof Error - ? new Error(`container-pool: slot ${index} was destroyed unused — ${error.message}`, { cause: error }) - : error - } - } - await normalizeContainerHome(opts, containerId, cli) - onProgress(`[slot ${index}] ready @ ${containerId.slice(0, 12)}`) - return { - containerId, - index, - busy: false, - dead: false, - lastSession: null, - holdTimer: null, - generation: 0, - consecutiveFailures: 0, - lastVerifiedAt: Date.now(), - armedStart, - } -} - -/** Fields that identify one container INCARNATION, in one `docker inspect`. */ -const START_FORMAT = '{{.State.Running}} {{.State.StartedAt}}' - -/** How many times arming may lose a race with a concurrent restart before it fails. */ -const ARM_ATTEMPTS = 3 - -/** - * Run `afterCreate` and return the container start it applies to. - * - * The start is read BEFORE and AFTER the hook, and the two must match. Reading - * it only afterwards would record the wrong thing in the one case that matters: - * a container that restarted DURING the hook would be stamped with the new - * start while the hook's work went into the namespace that is already gone — - * a slot that looks armed and is not. Losing the race repeatedly is reported as - * a failure rather than retried forever, because a container restarting that - * often is not a container to hand a request. - */ -async function armContainerStart( - opts: ContainerPoolOptions, - containerId: string, - index: number, - cli: DockerCli, -): Promise { - const afterCreate = opts.afterCreate - if (!afterCreate) return '' - let lastSeen = '' - for (let attempt = 0; attempt < ARM_ATTEMPTS; attempt++) { - const before = await readContainerStart(cli, containerId) - await afterCreate(containerId, index) - const after = await readContainerStart(cli, containerId) - if (after === before) return after - lastSeen = after - } - throw new Error( - `container-pool: slot ${index} restarted during afterCreate ${ARM_ATTEMPTS} times in a row ` + - `(last observed start ${lastSeen}); its per-start setup cannot be established`, - ) -} - -/** `.State.StartedAt` of a running container; throws if it is not running. */ -async function readContainerStart(cli: DockerCli, containerId: string): Promise { - const result = await cli(['inspect', '-f', START_FORMAT, containerId], { timeoutMs: 30_000 }) - const [running = '', startedAt = ''] = result.stdout.trim().split(/\s+/u) - if (result.code !== 0 || running !== 'true' || !startedAt) { - const detail = result.stderr.trim() || result.stdout.trim() || `docker exited ${result.code}` - throw new Error(`container ${containerId.slice(0, 12)} is not running — ${firstLine(detail)}`) - } - return startedAt -} - -/** - * Make a configured non-root HOME actually usable, as root inside the container. - * - * Docker creates the missing PARENTS of a bind mount as root:root. Mount - * credentials at `$HOME/.local/share/opencode` in an image that has no - * `$HOME/.local`, and Docker creates `$HOME/.local` and `$HOME/.local/share` - * owned by root — after which the uid-1000 CLI cannot create - * `$HOME/.local/state` and dies with `EACCES: permission denied, mkdir - * '/home/node/.local/state'`, an error that names a path nobody configured. - * Reproduced exactly on this host before this fix existed. - * - * So: create the XDG directories a CLI initializes and give the configured - * identity the ancestor chain of every mount target. Ownership is applied - * per-directory and never recursively, because a recursive chown would walk - * INTO the bind mounts and rewrite ownership of the operator's host files. - */ -async function normalizeContainerHome( - opts: ContainerPoolOptions, - containerId: string, - cli: DockerCli, -): Promise { - if (!opts.containerUser || !opts.containerHome) return - const home = opts.containerHome - const mountTargets = collectMountTargets(opts) - const chainDirs = new Set([home]) - for (const rel of ['.local', '.local/state', '.local/share', '.config', '.cache']) { - chainDirs.add(posixJoin(home, rel)) - } - for (const target of mountTargets) { - if (!isInside(home, target) || target === home) continue - // Every ancestor strictly between $HOME and the mount target, plus $HOME. - let cursor = posixDirname(target) - while (cursor.length >= home.length && isInside(home, cursor)) { - chainDirs.add(cursor) - if (cursor === home) break - cursor = posixDirname(cursor) - } - } - // Mount targets themselves are owned by whatever the host/volume provides; - // chowning them would mutate host files. - for (const target of mountTargets) chainDirs.delete(target) - - const quoted = [...chainDirs].map(shellQuote).join(' ') - const script = `set -e; mkdir -p ${quoted}; chown ${opts.containerUser} ${quoted}` - const result = await cli(containerShell(containerId, script, true), { timeoutMs: 30_000 }) - if (result.code !== 0) { - throw new Error( - `container-pool: cannot prepare HOME=${home} for ${opts.containerUser} inside ${opts.image} — ` + - `${firstLine(result.stderr) || `exit ${result.code}`}. Without a writable HOME the CLI fails at ` + - `request time with EACCES on a path you never configured.`, - ) - } -} - -function collectMountTargets(opts: ContainerPoolOptions): string[] { - const targets: string[] = [] - if (opts.oauthMode === 'share') { - for (const m of opts.shareMounts ?? []) { - const target = m.slice(m.indexOf(':') + 1) - if (target.startsWith('/')) targets.push(target) - } - } else { - for (const v of opts.perSlotVolumes ?? []) targets.push(v.target) - } - if (opts.workspaceRoot) targets.push(opts.workspaceRoot) - return targets -} - -/** Container paths are POSIX regardless of the host platform, so do not use node:path. */ -function posixDirname(path: string): string { - const idx = path.lastIndexOf('/') - if (idx <= 0) return '/' - return path.slice(0, idx) -} - -function posixJoin(base: string, rel: string): string { - return `${base.replace(/\/+$/u, '')}/${rel}` -} - -function shellQuote(value: string): string { - return `'${value.replace(/'/gu, `'\\''`)}'` -} - -function firstLine(text: string): string { - return text.split('\n').map((l) => l.trim()).find(Boolean) ?? '' -} - -/** Compose `docker run` argv without invoking Docker. */ -export function buildContainerRunArgs( - opts: ContainerPoolOptions, - index: number, - name = `${opts.namePrefix}-${index}`, -): string[] { - const memory = opts.memory ?? '4g' - const cpus = opts.cpus ?? '2' - const args = [ - 'run', '-d', - '--name', name, - '--restart', restartPolicyFor(opts), - '--memory', memory, '--memory-swap', memory, - '--cpus', cpus, - ] - if (opts.network !== undefined) { - args.push('--network', assertDockerNetworkName(opts.network)) - } - if (Boolean(opts.containerUser) !== Boolean(opts.containerHome)) { - throw new Error('container user and home must be configured together') - } - if (opts.containerUser && opts.containerHome) { - if (!/^[1-9][0-9]*:[1-9][0-9]*$/u.test(opts.containerUser)) { - throw new Error('invalid non-root container user') - } - if (!isSafeWorkspaceBindPath(opts.containerHome)) { - throw new Error('invalid non-root container home') - } - args.push('--user', opts.containerUser, '--env', `HOME=${opts.containerHome}`) - } - if (opts.workspaceRoot) { - if (!isSafeWorkspaceBindPath(opts.workspaceRoot)) { - throw new Error(`invalid Docker workspace root: ${opts.workspaceRoot}`) - } - // Docker bind mounts are read-write unless `readonly` is present. - // Source and target intentionally match so request cwd needs no rewrite. - args.push( - '--mount', - `type=bind,source=${opts.workspaceRoot},target=${opts.workspaceRoot}`, - ) - } - if (opts.oauthMode === 'share') { - for (const m of opts.shareMounts ?? []) args.push('-v', m) - } else { - if (!opts.perSlotVolumes || opts.perSlotVolumes.length === 0) { - throw new Error('per-slot oauthMode requires at least one perSlotVolumes entry') - } - for (const v of opts.perSlotVolumes) args.push('-v', `${v.volumePrefix}-${index}:${v.target}`) - } - args.push(opts.image, 'tail', '-f', '/dev/null') - return args -} - -/** - * The Docker restart policy a slot runs under. - * - * Without `afterCreate`, `on-failure:3` instead of `unless-stopped` — it caps - * the docker-level restart loop so a poisoned image can't churn the daemon - * forever, and reviving the container in place is a strictly cheaper recovery - * than rebuilding it. - * - * WITH `afterCreate`, none at all. A restart policy revives a container with - * fresh, empty namespaces — the setup the hook wrote is gone, the container id - * and every mount are unchanged, and the revival happens with nothing in this - * process observing it. The pool already models a dead slot correctly (`rm -f` - * then `provisionSlot`, which re-runs the hook), so the honest configuration is - * for a dead jailed container to be REPLACED, never revived. `tail -f /dev/null` - * is the entrypoint, so a slot only dies when something killed it, which is a - * container to discard rather than to bring back. - * - * This is one half of the invariant; the other is `ensureSlotUsable`, which - * covers the restarts this bridge performs ITSELF — `terminateDockerExecution` - * restarts a slot to kill a worker's process tree whenever the worker exits - * non-cleanly, which a worker triggers simply by exiting non-zero. Removing the - * policy without that gate would have closed the smaller of the two doors. - */ -function restartPolicyFor(opts: ContainerPoolOptions): string { - return opts.afterCreate ? 'no' : 'on-failure:3' -} - -function isSafeWorkspaceBindPath(path: string): boolean { - return path.startsWith('/') && path !== '/' && !path.includes(',') -} - -async function destroySlot(containerId: string): Promise { - if (!containerId) return - await execFileAsync('docker', ['rm', '-f', containerId]).catch(() => {}) -} diff --git a/src/executors/container-slot.ts b/src/executors/container-slot.ts new file mode 100644 index 0000000..637e9b6 --- /dev/null +++ b/src/executors/container-slot.ts @@ -0,0 +1,293 @@ +/** Docker slot provisioning, start-scoped setup and run configuration. */ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import type { ContainerPoolOptions, SlotState } from './container-pool.js' +import { assertDockerNetworkName } from './docker-network.js' +import { containerShell, dockerCli, type DockerCli } from './docker-cli.js' +import { buildCommandFor, isInside } from './docker-preflight.js' +const execFileAsync = promisify(execFile) + +export async function provisionSlot( + opts: ContainerPoolOptions, + index: number, + onProgress: (m: string) => void, +): Promise { + const name = `${opts.namePrefix}-${index}` + // Tear down any stale container with the same name (idempotent). + await execFileAsync('docker', ['rm', '-f', name]).catch(() => {}) + + const args = buildContainerRunArgs(opts, index, name) + + onProgress(`[slot ${index}] docker run ${name}`) + const cli = opts.cli ?? dockerCli + const r = await cli(args, { timeoutMs: 120_000 }) + const containerId = r.stdout.trim() + if (r.code !== 0 || !containerId) { + // A missing image is the single most common cause and Docker reports it as + // a registry/pull problem. Name the image and the build command so this + // never again surfaces later as a message about a missing CONTAINER. + const stderr = r.stderr.trim() || r.spawnError || `docker exited ${r.code}` + const missingImage = /No such image|pull access denied|manifest unknown|not found: manifest|Unable to find image/i.test(stderr) + throw new Error( + missingImage + ? `container-pool: cannot create slot ${index} — image ${opts.image} is not available on this host. ` + + `Build it: ${buildCommandFor(opts.image)}. (docker: ${firstLine(stderr)})` + : `container-pool: cannot create slot ${index} — ${firstLine(stderr)}`, + ) + } + // Before anything else touches the container, including this bridge's own + // `docker exec`: the window between `run` and the filter being in place is + // the only moment a slot has unrestricted egress, and the only thing running + // in it is the idle `tail -f /dev/null` entrypoint. + let armedStart = '' + if (opts.afterCreate) { + try { + armedStart = await armContainerStart(opts, containerId, index, cli) + } catch (error) { + await cli(['rm', '-f', containerId], { timeoutMs: 30_000 }) + throw error instanceof Error + ? new Error(`container-pool: slot ${index} was destroyed unused — ${error.message}`, { cause: error }) + : error + } + } + await normalizeContainerHome(opts, containerId, cli) + onProgress(`[slot ${index}] ready @ ${containerId.slice(0, 12)}`) + return { + containerId, + index, + busy: false, + dead: false, + lastSession: null, + holdTimer: null, + generation: 0, + consecutiveFailures: 0, + lastVerifiedAt: Date.now(), + armedStart, + } +} + +/** Fields that identify one container INCARNATION, in one `docker inspect`. */ +export const START_FORMAT = '{{.State.Running}} {{.State.StartedAt}}' + +/** How many times arming may lose a race with a concurrent restart before it fails. */ +const ARM_ATTEMPTS = 3 + +/** + * Run `afterCreate` and return the container start it applies to. + * + * The start is read BEFORE and AFTER the hook, and the two must match. Reading + * it only afterwards would record the wrong thing in the one case that matters: + * a container that restarted DURING the hook would be stamped with the new + * start while the hook's work went into the namespace that is already gone — + * a slot that looks armed and is not. Losing the race repeatedly is reported as + * a failure rather than retried forever, because a container restarting that + * often is not a container to hand a request. + */ +export async function armContainerStart( + opts: ContainerPoolOptions, + containerId: string, + index: number, + cli: DockerCli, +): Promise { + const afterCreate = opts.afterCreate + if (!afterCreate) return '' + let lastSeen = '' + for (let attempt = 0; attempt < ARM_ATTEMPTS; attempt++) { + const before = await readContainerStart(cli, containerId) + await afterCreate(containerId, index) + const after = await readContainerStart(cli, containerId) + if (after === before) return after + lastSeen = after + } + throw new Error( + `container-pool: slot ${index} restarted during afterCreate ${ARM_ATTEMPTS} times in a row ` + + `(last observed start ${lastSeen}); its per-start setup cannot be established`, + ) +} + +/** `.State.StartedAt` of a running container; throws if it is not running. */ +async function readContainerStart(cli: DockerCli, containerId: string): Promise { + const result = await cli(['inspect', '-f', START_FORMAT, containerId], { timeoutMs: 30_000 }) + const [running = '', startedAt = ''] = result.stdout.trim().split(/\s+/u) + if (result.code !== 0 || running !== 'true' || !startedAt) { + const detail = result.stderr.trim() || result.stdout.trim() || `docker exited ${result.code}` + throw new Error(`container ${containerId.slice(0, 12)} is not running — ${firstLine(detail)}`) + } + return startedAt +} + +/** + * Make a configured non-root HOME actually usable, as root inside the container. + * + * Docker creates the missing PARENTS of a bind mount as root:root. Mount + * credentials at `$HOME/.local/share/opencode` in an image that has no + * `$HOME/.local`, and Docker creates `$HOME/.local` and `$HOME/.local/share` + * owned by root — after which the uid-1000 CLI cannot create + * `$HOME/.local/state` and dies with `EACCES: permission denied, mkdir + * '/home/node/.local/state'`, an error that names a path nobody configured. + * Reproduced exactly on this host before this fix existed. + * + * So: create the XDG directories a CLI initializes and give the configured + * identity the ancestor chain of every mount target. Ownership is applied + * per-directory and never recursively, because a recursive chown would walk + * INTO the bind mounts and rewrite ownership of the operator's host files. + */ +async function normalizeContainerHome( + opts: ContainerPoolOptions, + containerId: string, + cli: DockerCli, +): Promise { + if (!opts.containerUser || !opts.containerHome) return + const home = opts.containerHome + const mountTargets = collectMountTargets(opts) + const chainDirs = new Set([home]) + for (const rel of ['.local', '.local/state', '.local/share', '.config', '.cache']) { + chainDirs.add(posixJoin(home, rel)) + } + for (const target of mountTargets) { + if (!isInside(home, target) || target === home) continue + // Every ancestor strictly between $HOME and the mount target, plus $HOME. + let cursor = posixDirname(target) + while (cursor.length >= home.length && isInside(home, cursor)) { + chainDirs.add(cursor) + if (cursor === home) break + cursor = posixDirname(cursor) + } + } + // Mount targets themselves are owned by whatever the host/volume provides; + // chowning them would mutate host files. + for (const target of mountTargets) chainDirs.delete(target) + + const quoted = [...chainDirs].map(shellQuote).join(' ') + const script = `set -e; mkdir -p ${quoted}; chown ${opts.containerUser} ${quoted}` + const result = await cli(containerShell(containerId, script, true), { timeoutMs: 30_000 }) + if (result.code !== 0) { + throw new Error( + `container-pool: cannot prepare HOME=${home} for ${opts.containerUser} inside ${opts.image} — ` + + `${firstLine(result.stderr) || `exit ${result.code}`}. Without a writable HOME the CLI fails at ` + + `request time with EACCES on a path you never configured.`, + ) + } +} + +function collectMountTargets(opts: ContainerPoolOptions): string[] { + const targets: string[] = [] + if (opts.oauthMode === 'share') { + for (const m of opts.shareMounts ?? []) { + const target = m.slice(m.indexOf(':') + 1) + if (target.startsWith('/')) targets.push(target) + } + } else { + for (const v of opts.perSlotVolumes ?? []) targets.push(v.target) + } + if (opts.workspaceRoot) targets.push(opts.workspaceRoot) + return targets +} + +/** Container paths are POSIX regardless of the host platform, so do not use node:path. */ +function posixDirname(path: string): string { + const idx = path.lastIndexOf('/') + if (idx <= 0) return '/' + return path.slice(0, idx) +} + +function posixJoin(base: string, rel: string): string { + return `${base.replace(/\/+$/u, '')}/${rel}` +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/gu, `'\\''`)}'` +} + +function firstLine(text: string): string { + return text.split('\n').map((l) => l.trim()).find(Boolean) ?? '' +} + +/** Compose `docker run` argv without invoking Docker. */ +export function buildContainerRunArgs( + opts: ContainerPoolOptions, + index: number, + name = `${opts.namePrefix}-${index}`, +): string[] { + const memory = opts.memory ?? '4g' + const cpus = opts.cpus ?? '2' + const args = [ + 'run', '-d', + '--name', name, + '--restart', restartPolicyFor(opts), + '--memory', memory, '--memory-swap', memory, + '--cpus', cpus, + ] + if (opts.network !== undefined) { + args.push('--network', assertDockerNetworkName(opts.network)) + } + if (Boolean(opts.containerUser) !== Boolean(opts.containerHome)) { + throw new Error('container user and home must be configured together') + } + if (opts.containerUser && opts.containerHome) { + if (!/^[1-9][0-9]*:[1-9][0-9]*$/u.test(opts.containerUser)) { + throw new Error('invalid non-root container user') + } + if (!isSafeWorkspaceBindPath(opts.containerHome)) { + throw new Error('invalid non-root container home') + } + args.push('--user', opts.containerUser, '--env', `HOME=${opts.containerHome}`) + } + if (opts.workspaceRoot) { + if (!isSafeWorkspaceBindPath(opts.workspaceRoot)) { + throw new Error(`invalid Docker workspace root: ${opts.workspaceRoot}`) + } + // Docker bind mounts are read-write unless `readonly` is present. + // Source and target intentionally match so request cwd needs no rewrite. + args.push( + '--mount', + `type=bind,source=${opts.workspaceRoot},target=${opts.workspaceRoot}`, + ) + } + if (opts.oauthMode === 'share') { + for (const m of opts.shareMounts ?? []) args.push('-v', m) + } else { + if (!opts.perSlotVolumes || opts.perSlotVolumes.length === 0) { + throw new Error('per-slot oauthMode requires at least one perSlotVolumes entry') + } + for (const v of opts.perSlotVolumes) args.push('-v', `${v.volumePrefix}-${index}:${v.target}`) + } + args.push(opts.image, 'tail', '-f', '/dev/null') + return args +} + +/** + * The Docker restart policy a slot runs under. + * + * Without `afterCreate`, `on-failure:3` instead of `unless-stopped` — it caps + * the docker-level restart loop so a poisoned image can't churn the daemon + * forever, and reviving the container in place is a strictly cheaper recovery + * than rebuilding it. + * + * WITH `afterCreate`, none at all. A restart policy revives a container with + * fresh, empty namespaces — the setup the hook wrote is gone, the container id + * and every mount are unchanged, and the revival happens with nothing in this + * process observing it. The pool already models a dead slot correctly (`rm -f` + * then `provisionSlot`, which re-runs the hook), so the honest configuration is + * for a dead jailed container to be REPLACED, never revived. `tail -f /dev/null` + * is the entrypoint, so a slot only dies when something killed it, which is a + * container to discard rather than to bring back. + * + * This is one half of the invariant; the other is `ensureSlotUsable`, which + * covers the restarts this bridge performs ITSELF — `terminateDockerExecution` + * restarts a slot to kill a worker's process tree whenever the worker exits + * non-cleanly, which a worker triggers simply by exiting non-zero. Removing the + * policy without that gate would have closed the smaller of the two doors. + */ +function restartPolicyFor(opts: ContainerPoolOptions): string { + return opts.afterCreate ? 'no' : 'on-failure:3' +} + +function isSafeWorkspaceBindPath(path: string): boolean { + return path.startsWith('/') && path !== '/' && !path.includes(',') +} + +export async function destroySlot(containerId: string): Promise { + if (!containerId) return + await execFileAsync('docker', ['rm', '-f', containerId]).catch(() => {}) +} diff --git a/src/executors/docker-runtime-roll.ts b/src/executors/docker-runtime-roll.ts new file mode 100644 index 0000000..c0418dd --- /dev/null +++ b/src/executors/docker-runtime-roll.ts @@ -0,0 +1,185 @@ +/** Immutable runtime replacement. Docker configuration stays in memory, never in logs. */ +import { request } from 'node:http' +import { readFileSync } from 'node:fs' +import { dockerCli } from './docker-cli.js' +import { preflightDockerSlot } from './docker-preflight.js' + +type Json = Record +export async function dockerEngine(method: string, path: string, body?: unknown): Promise { + const host = process.env.DOCKER_HOST ?? 'unix:///var/run/docker.sock' + if (!host.startsWith('unix://')) throw new Error('maintenance requires a local Docker socket') + return await new Promise((resolve, reject) => { + const req = request({ socketPath: host.slice(7), path, method, + headers: { 'content-type': 'application/json' }, timeout: 120_000 }, (res) => { + let data = '' + res.setEncoding('utf8') + res.on('data', (part: string) => { data += part; if (data.length > 8 * 1024 * 1024) req.destroy() }) + res.on('end', () => { + if (!res.statusCode || res.statusCode >= 300) reject(new Error(`Docker maintenance request refused (${res.statusCode})`)) + else { try { resolve(data ? JSON.parse(data) : null) } catch { reject(new Error('invalid Docker response')) } } + }) + }) + req.on('timeout', () => req.destroy(new Error('Docker maintenance timeout'))) + req.on('error', () => reject(new Error('Docker maintenance transport failed'))) + req.end(body === undefined ? undefined : JSON.stringify(body)) + }) +} + +export async function inspectIdle(id: string, expectedName?: string): Promise { + if (!/^[a-f0-9]{64}$/u.test(id)) throw new Error('full container identity required') + const info = await dockerEngine('GET', `/containers/${id}/json`) + if (info.Id !== id || !info.State.Running || info.State.Paused || info.State.Restarting || + info.ExecIDs?.length || (expectedName && info.Name !== `/${expectedName}`) || + JSON.stringify(info.Config.Cmd) !== JSON.stringify(['tail', '-f', '/dev/null']) || + JSON.stringify(info.Config.Entrypoint) !== JSON.stringify(['dumb-init', '--'])) { + throw new Error('container is not the expected idle CLI slot') + } + // Docker itself requires PID in ps output to filter the container's tasks. + const top = await dockerCli(['top', id, '-eo', 'pid,ppid,comm']) + const rows = top.stdout.trim().split('\n').slice(1).map((line) => line.trim().split(/\s+/u)) + const commands = rows.map((row) => row[2] ?? '') + const init = rows.find((row) => row[2] === 'dumb-init') + const tail = rows.find((row) => row[2] === 'tail') + if (top.code !== 0 || rows.length !== 2 || rows.some((row) => row.length !== 3 || !/^\d+$/u.test(row[0]!) || !/^\d+$/u.test(row[1]!)) || + !init || !tail || Number(init[0]) !== info.State.Pid || tail[1] !== init[0]) { + const observed = commands.map((name) => /^[a-zA-Z0-9._-]{1,64}$/u.test(name) ? name : '') + throw new Error(`slot contains unexpected processes (${JSON.stringify(observed)}); leave it drained until its work finishes`) + } + return info +} + +function envMap(values: string[]): Map { + return new Map(values.map((value) => [value.slice(0, value.indexOf('=')), value])) +} + +/** Keep operator overrides; image-owned environment follows the patched image. */ +function replacementEnvironment(old: Json, oldImage: Json, image: Json): string[] { + const baseline = envMap(oldImage.Config.Env ?? []) + const values = envMap(image.Config.Env ?? []) + for (const [key, value] of envMap(old.Config.Env ?? [])) { + if (baseline.get(key) !== value) values.set(key, value) + } + return [...values.values()] +} + +function cloneConfiguration(old: Json, oldImage: Json, image: Json): Json { + const host = structuredClone(old.HostConfig) + // Sharing another container's namespaces or devices is outside pool ownership. + if (!['default', 'bridge'].includes(host.NetworkMode) || host.Privileged || host.AutoRemove || + host.ContainerIDFile || host.VolumesFrom?.length || host.Links?.length || + Object.keys(old.NetworkSettings.Networks ?? {}).some((name) => name !== 'bridge') || + old.Config.Domainname || host.PidMode || host.IpcMode === 'host' || host.Devices?.length || + Object.keys(host.PortBindings ?? {}).length || old.Mounts.some((m: Json) => !['bind', 'volume'].includes(m.Type))) { + throw new Error('unsupported slot namespace, device, port or mount topology') + } + host.Binds = null + host.VolumesFrom = null + host.Mounts = old.Mounts.map((m: Json) => ({ Type: m.Type, + Source: m.Type === 'volume' ? m.Name : m.Source, Target: m.Destination, ReadOnly: !m.RW, + ...(m.Type === 'bind' ? { BindOptions: { Propagation: m.Propagation || 'rprivate' } } : + { VolumeOptions: { NoCopy: true } }) })) + return { ...old.Config, Hostname: '', Domainname: '', Image: image.Id, + Env: replacementEnvironment(old, oldImage, image), Volumes: {}, HostConfig: host } +} + +/** Execute the maintained runtime pin policy, then the normal Docker preflight. */ +export async function verifyRuntime(id: string, imageId: string, original: Json): Promise { + const dockerfile = readFileSync(new URL('../../docker/Dockerfile.cli-runtime', import.meta.url), 'utf8') + const required = /^FROM node:(\d+\.\d+\.\d+)-/mu.exec(dockerfile)?.[1] + if (!required) throw new Error('runtime Node policy is unavailable') + const result = await dockerCli(['exec', id, 'node', '--version']) + if (result.code !== 0 || result.stdout.trim() !== `v${required}`) throw new Error('candidate fails maintained Node security policy') + const candidate = await dockerEngine('GET', `/containers/${id}/json`) + for (const mount of original.Mounts) { + if (!candidate.Mounts.some((m: Json) => m.Type === mount.Type && m.Destination === mount.Destination && + m.RW === mount.RW && (m.Type === 'volume' ? m.Name === mount.Name : + m.Source === mount.Source && m.Propagation === mount.Propagation))) { + throw new Error('candidate changed an existing mount') + } + } + for (const key of ['User', 'WorkingDir', 'Entrypoint', 'Cmd']) { + if (JSON.stringify(candidate.Config[key]) !== JSON.stringify(original.Config[key])) throw new Error('candidate changed process configuration') + } + const home = (original.Config.Env ?? []).find((v: string) => v.startsWith('HOME='))?.slice(5) ?? '/root' + const workspace = original.Mounts.find((m: Json) => m.Type === 'bind' && m.Source === m.Destination)?.Destination + const findings = await preflightDockerSlot({ backend: 'runtime', envPrefix: 'BRIDGE', image: imageId, + workspaceRoot: workspace, bin: 'node', containerHome: home, containerUser: original.Config.User || undefined, + mounts: original.Mounts.filter((m: Json) => m.Destination.startsWith(`${home}/`)).map((m: Json) => ({ source: m.Type === 'volume' ? m.Name : m.Source, + target: m.Destination, kind: m.Type })), buildCommand: 'pnpm run docker:build:runtime' }, id, dockerCli) + if (findings.length) throw new Error('candidate fails maintained Docker preflight') + await inspectIdle(id) + return result.stdout.trim() +} + +export interface RollReceipt { + oldId: string + imageId: string + name: string + candidateId?: string + retainedName?: string + phase: 'intent' | 'candidate' | 'verified' | 'stopped' | 'retained' | 'promoted' + node?: string + nonce: string +} + +/** Each irreversible boundary is journaled; retries use exact recorded IDs. */ +export async function replaceRuntime(receipt: RollReceipt, save: () => void, arm: (id: string) => Promise): Promise { + if (!/^sha256:[a-f0-9]{64}$/u.test(receipt.imageId)) throw new Error('immutable local image identity required') + const image = await dockerEngine('GET', `/images/${receipt.imageId}/json`) + if (image.Id !== receipt.imageId) throw new Error('candidate image identity changed') + const old = await dockerEngine('GET', `/containers/${receipt.oldId}/json`) + if (receipt.phase === 'intent') { + await inspectIdle(receipt.oldId, receipt.name) + const oldImage = await dockerEngine('GET', `/images/${old.Image}/json`) + // Name is deterministic, so a response lost after creation can be read back. + const candidateName = `${receipt.name}-candidate-${receipt.nonce}` + let candidate: Json + try { candidate = await dockerEngine('GET', `/containers/${candidateName}/json`) } + catch { + const config = cloneConfiguration(old, oldImage, image) + config.Labels = { ...config.Labels, 'cli-bridge.maintenance.nonce': receipt.nonce, 'cli-bridge.maintenance.old-id': receipt.oldId } + candidate = await dockerEngine('POST', `/containers/create?name=${candidateName}`, config) + } + receipt.candidateId = candidate.Id + receipt.phase = 'candidate'; save() + } + const id = receipt.candidateId! + const candidate = await dockerEngine('GET', `/containers/${id}/json`) + if (candidate.Config.Labels?.['cli-bridge.maintenance.nonce'] !== receipt.nonce || + candidate.Config.Labels?.['cli-bridge.maintenance.old-id'] !== receipt.oldId || candidate.Image !== image.Id) throw new Error('candidate image mismatch') + if (receipt.phase === 'candidate') { + if (!candidate.State.Running) await dockerEngine('POST', `/containers/${id}/start`) + await arm(id) + receipt.node = await verifyRuntime(id, image.Id, old) + receipt.phase = 'verified'; save() + } + if (receipt.phase === 'verified') { + await arm(id) + await verifyRuntime(id, image.Id, old) + if (old.State.Running) { + await inspectIdle(receipt.oldId, receipt.name) + await dockerEngine('POST', `/containers/${receipt.oldId}/stop?t=10`) + } + receipt.phase = 'stopped'; save() + } + receipt.retainedName ??= `${receipt.name}-retained-${receipt.oldId.slice(0, 12)}` + if (receipt.phase === 'stopped') { + const current = await dockerEngine('GET', `/containers/${receipt.oldId}/json`) + if (current.State.Running) throw new Error('old slot restarted during promotion') + if (current.Name !== `/${receipt.retainedName}`) await dockerEngine('POST', `/containers/${receipt.oldId}/rename?name=${receipt.retainedName}`) + receipt.phase = 'retained'; save() + } + if (receipt.phase === 'retained') { + await arm(id) + await verifyRuntime(id, image.Id, old) + const current = await dockerEngine('GET', `/containers/${id}/json`) + if (current.Name !== `/${receipt.name}`) await dockerEngine('POST', `/containers/${id}/rename?name=${receipt.name}`) + receipt.phase = 'promoted'; save() + } + await arm(id) + await verifyRuntime(id, image.Id, old) + await inspectIdle(id, receipt.name) + const retained = await dockerEngine('GET', `/containers/${receipt.oldId}/json`) + if (retained.State.Running) throw new Error('retained old container unexpectedly running') + return id +} diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts index c83e398..d6254ab 100644 --- a/src/routes/metrics.ts +++ b/src/routes/metrics.ts @@ -15,7 +15,7 @@ import { hostExecutorSnapshot } from '../executors/host.js' import { scopedHostExecutorSnapshot } from '../executors/scoped-host.js' import type { ContainerPool } from '../executors/container-pool.js' -const registeredPools = new Map() +export const registeredPools = new Map() export function registerPoolForMetrics(name: string, pool: ContainerPool): void { registeredPools.set(name, pool) diff --git a/src/routes/pool-maintenance.ts b/src/routes/pool-maintenance.ts new file mode 100644 index 0000000..1e57533 --- /dev/null +++ b/src/routes/pool-maintenance.ts @@ -0,0 +1,39 @@ +/** Authenticated, explicit ownership transfer and individually drained runtime roll. */ +import type { Hono } from 'hono' +import { z } from 'zod' +import { adoptionRequest, PoolMaintenance } from '../runtime/pool-maintenance.js' + +const rollRequest = z.object({ expectedId: z.string().regex(/^[a-f0-9]{64}$/u), + imageId: z.string().regex(/^sha256:[a-f0-9]{64}$/u) }).strict() +const drainRequest = rollRequest.omit({ imageId: true }) + +export function mountPoolMaintenance(app: Hono, owner: PoolMaintenance, authenticated: boolean): void { + app.use('/admin/pools/*', async (c, next) => { + if (!authenticated) return c.json({ error: 'pool maintenance requires configured bearer authentication' }, 403) + return next() + }) + app.get('/admin/pools/status', (c) => c.json({ pools: owner.snapshot() })) + app.post('/admin/pools/adopt', async (c) => { + try { + const text = await c.req.text() + if (text.length > 32_768) return c.json({ error: 'request too large' }, 413) + return c.json({ pools: await owner.adopt(adoptionRequest.parse(JSON.parse(text))) }) + } catch { return c.json({ error: 'adoption refused; preserve existing containers and inspect ownership' }, 409) } + }) + app.post('/admin/pools/:prefix/slots/:index/:operation', async (c) => { + try { + const text = await c.req.text() + if (text.length > 4096) return c.json({ error: 'request too large' }, 413) + const index = Number(c.req.param('index')) + if (!Number.isInteger(index) || index < 0 || index > 31) throw new Error('invalid slot') + const name = c.req.param('prefix') + if (c.req.param('operation') === 'drain') { + const input = drainRequest.parse(JSON.parse(text)) + return c.json({ slots: owner.drain(name, index, input.expectedId) }) + } + if (c.req.param('operation') !== 'roll') return c.json({ error: 'unknown operation' }, 404) + const input = rollRequest.parse(JSON.parse(text)) + return c.json({ receipt: await owner.roll(name, index, input.expectedId, input.imageId) }) + } catch { return c.json({ error: 'slot maintenance refused; inspect drain, active work and durable receipt' }, 409) } + }) +} diff --git a/src/runtime/pool-maintenance.ts b/src/runtime/pool-maintenance.ts new file mode 100644 index 0000000..cf90553 --- /dev/null +++ b/src/runtime/pool-maintenance.ts @@ -0,0 +1,181 @@ +/** Durable adoption and one-slot runtime rolls, owned by the bridge's admin API. */ +import { randomBytes } from 'node:crypto' +import { closeSync, constants, fsyncSync, fstatSync, openSync, readFileSync, renameSync, realpathSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { createConnection } from 'node:net' +import { z } from 'zod' +import { ContainerPool } from '../executors/container-pool.js' +import { inspectIdle, replaceRuntime, type RollReceipt } from '../executors/docker-runtime-roll.js' +import { registeredPools } from '../routes/metrics.js' +import { acquireDataDirectoryLock, type DataDirectoryLock } from './single-instance.js' + +const identity = z.string().regex(/^[a-f0-9]{64}$/u) +const prefix = z.string().regex(/^cli-bridge-[a-zA-Z0-9_-]{1,64}$/u) +export const adoptionRequest = z.object({ prefix, legacyDataDir: z.string().startsWith('/'), + expectedImage: z.string().regex(/^sha256:[a-f0-9]{64}$/u), + legacyPorts: z.array(z.number().int().min(1).max(65535)).min(1).max(4), + slots: z.array(z.object({ index: z.number().int().min(0).max(31), id: identity }).strict()).min(1).max(32), +}).strict() +const rollSchema = z.object({ oldId: identity, imageId: z.string().regex(/^sha256:[a-f0-9]{64}$/u), name: z.string(), + nonce: z.string().regex(/^[a-f0-9]{32}$/u), candidateId: identity.optional(), retainedName: z.string().optional(), + phase: z.enum(['intent', 'candidate', 'verified', 'stopped', 'retained', 'promoted']), node: z.string().optional(), +}).strict() +const recordSchema = adoptionRequest.extend({ rolls: z.record(z.string(), rollSchema), image: z.string() }).strict() +type Record = z.infer + +async function requireNoListener(port: number): Promise { + await new Promise((resolve, reject) => { + const socket = createConnection({ host: '127.0.0.1', port }) + socket.setTimeout(1000) + socket.once('connect', () => { socket.destroy(); reject(new Error('legacy controller still listens')) }) + socket.once('timeout', () => { socket.destroy(); reject(new Error('legacy controller state unknown')) }) + socket.once('error', (error: NodeJS.ErrnoException) => { + if (error.code === 'ECONNREFUSED') resolve() + else reject(new Error('legacy controller state unknown')) + }) + }) +} + +export class PoolMaintenance { + private records: Record[] = [] + private locks = new Map() + private pools = new Map() + private operating = new Set() + private closing = false + private path: string + + constructor(dataDir: string) { + this.path = join(dataDir, 'pool-maintenance.json') + try { + const fd = openSync(this.path, constants.O_RDONLY | constants.O_NOFOLLOW) + try { + const stat = fstatSync(fd) + if (!stat.isFile() || stat.uid !== process.getuid?.() || (stat.mode & 0o077)) throw new Error('unsafe maintenance journal') + this.records = z.object({ schema: z.literal(1), pools: z.array(recordSchema).max(32) }).strict() + .parse(JSON.parse(readFileSync(fd, 'utf8'))).pools + } finally { closeSync(fd) } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw new Error('maintenance journal needs operator reconciliation') + } + for (const record of this.records) { + this.lock(record.legacyDataDir) + this.restore(record) + } + } + + private lock(dir: string): void { + const canonical = realpathSync(dir) + if (!this.locks.has(canonical)) this.locks.set(canonical, acquireDataDirectoryLock(canonical)) + } + + private restore(record: Record): ContainerPool { + if (this.pools.has(record.prefix)) return this.pools.get(record.prefix)! + const ids = record.slots.map((slot) => { + const roll = record.rolls[String(slot.index)] + return roll?.phase === 'promoted' ? roll.candidateId! : slot.id + }) + const pool = ContainerPool.adopt({ size: ids.length, image: record.image, + namePrefix: record.prefix, oauthMode: 'share', slotMaxHoldMs: 0 }, ids) + this.pools.set(record.prefix, pool) + registeredPools.set(record.prefix, pool) + return pool + } + + private save(): void { + const temp = `${this.path}.${randomBytes(8).toString('hex')}.next` + const fd = openSync(temp, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) + try { writeFileSync(fd, JSON.stringify({ schema: 1, pools: this.records })); fsyncSync(fd) } + finally { closeSync(fd) } + renameSync(temp, this.path) + const dir = openSync(dirname(this.path), constants.O_RDONLY | constants.O_DIRECTORY) + try { fsyncSync(dir) } finally { closeSync(dir) } + } + + snapshot() { + return [...this.pools].map(([name, pool]) => ({ prefix: name, slots: pool.slotSnapshot(), + rolls: Object.values(this.records.find((r) => r.prefix === name)!.rolls) })) + } + + async adopt(input: z.infer) { + if (this.closing || this.operating.size) throw new Error('another maintenance operation is active') + this.operating.add(input.prefix) + try { + if (this.records.some((r) => r.prefix === input.prefix)) throw new Error('pool already adopted; inspect its durable receipt') + if (input.slots.some((slot, index) => slot.index !== index) || new Set(input.slots.map((s) => s.id)).size !== input.slots.length) { + throw new Error('slots must be ordered contiguous distinct identities') + } + for (const pool of registeredPools.values()) { + if (input.slots.some((s) => pool.liveContainerIds().includes(s.id))) throw new Error('slot is already owned by a live pool') + } + this.lock(input.legacyDataDir) + for (const port of input.legacyPorts) await requireNoListener(port) + const root = `${dirname(realpathSync(input.legacyDataDir))}/` + let image = '' + for (const slot of input.slots) { + const info = await inspectIdle(slot.id, `${input.prefix}-${slot.index}`) + if (!info.Mounts.some((mount: any) => mount.Type === 'bind' && mount.Source.startsWith(root))) { + throw new Error('slot has no bind association with the locked legacy data directory') + } + if (info.Image !== input.expectedImage) throw new Error('adoption image identity differs from explicit manifest') + if (image && image !== info.Image) throw new Error('adoption requires one known runtime image') + image = info.Image + } + const record = { ...input, legacyDataDir: realpathSync(input.legacyDataDir), image, rolls: {} } + this.records.push(record) + this.save() + this.restore(record) + return this.snapshot() + } finally { + this.operating.delete(input.prefix) + const canonical = realpathSync(input.legacyDataDir) + if (!this.records.some((record) => record.legacyDataDir === canonical)) { + this.locks.get(canonical)?.release() + this.locks.delete(canonical) + } + } + } + + drain(name: string, index: number, expectedId: string) { + const pool = this.pools.get(name) + if (this.closing || !pool || this.operating.has(name)) throw new Error('pool unavailable or maintenance is active') + pool.drainSlot(index, expectedId) + return pool.slotSnapshot() + } + + async roll(name: string, index: number, expectedId: string, imageId: string) { + const pool = this.pools.get(name) + const record = this.records.find((r) => r.prefix === name) + if (this.closing || !pool || !record || this.operating.size) throw new Error('pool unavailable or maintenance is active') + const slot = record.slots[index] + if (!slot || slot.index !== index) throw new Error('unknown slot') + const previous = record.rolls[String(index)] + if (previous && (previous.oldId !== expectedId || previous.imageId !== imageId)) throw new Error('roll intent differs from durable receipt') + const receipt: RollReceipt = previous ?? { oldId: expectedId, imageId, + name: `${name}-${index}`, nonce: randomBytes(16).toString('hex'), phase: 'intent' } + this.operating.add(name) + try { + if (previous?.phase === 'promoted') { + const current = pool.slotSnapshot()[index]! + pool.drainSlot(index, current.containerId) + await pool.rollSlot(index, current.containerId, imageId, async (_id, arm) => + await replaceRuntime(previous, () => this.save(), arm)) + return previous + } + // Persist the identity before the first Docker write; failures stay drained. + record.rolls[String(index)] = receipt + this.save() + await pool.rollSlot(index, expectedId, imageId, async (_id, arm) => + await replaceRuntime(receipt, () => this.save(), arm)) + record.image = imageId + this.save() + return receipt + } finally { this.operating.delete(name) } + } + + async close(): Promise { + this.closing = true + while (this.operating.size) await new Promise((resolve) => setTimeout(resolve, 50)) + for (const lock of this.locks.values()) lock.release() + this.locks.clear() + } +} diff --git a/src/server.ts b/src/server.ts index 3907aec..2b50b00 100644 --- a/src/server.ts +++ b/src/server.ts @@ -39,6 +39,8 @@ import { RetainedSessionService, mountRetainedSessions } from './sessions/retain import { mountCadRender } from './routes/cad-render.js' import { mountImagesGenerate } from './routes/images-generate.js' import { mountMetrics, registerPoolForMetrics } from './routes/metrics.js' +import { mountPoolMaintenance } from './routes/pool-maintenance.js' +import { PoolMaintenance } from './runtime/pool-maintenance.js' import { ContainerPool } from './executors/container-pool.js' import { provisionNetJail } from './executors/net-jail-network.js' import { modelEndpointsFor, parseAllowList } from './jail/net-allowlist.js' @@ -535,6 +537,9 @@ export async function buildApp(config: Config): Promise<{ mountCadRender(app) mountImagesGenerate(app) mountMetrics(app) + const poolMaintenance = new PoolMaintenance(config.dataDir) + extras.shutdownHooks.push(async () => await poolMaintenance.close()) + mountPoolMaintenance(app, poolMaintenance, Boolean(config.bearer)) app.get('/', (c) => c.json({ name: 'cli-bridge',