Provide persistent backend deployment and current object API guides

.github/scripts/backend-docs.test.mjs

@@ -0,0 +1,44 @@
+import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { existsSync, readFileSync } from 'node:fs'
+import { dirname, resolve } from 'node:path'
+import { test } from 'node:test'
+
+const root = resolve(import.meta.dirname, '../..')
+const read = name => readFileSync(resolve(root, name), 'utf8')
+const docs = ['README.md', 'README.zh.md', 'docs/backend-operations.md', 'docs/development-service-api.md', 'docs/development-service-guide.md']
+const names = text => [...text.matchAll(/'([^']+)'/g)].map(match => match[1])
+
+test('public API tables enumerate the actual complete command/query contracts', () => {
+  const source = read('core/src/host/project-development-service.js')
+  const queries = names(source.match(/const queries = new Set\(\[([^\]]+)\]/)[1])
+  const operations = [...source.match(/const operations = Object.freeze\(\{([\s\S]*?)\}\)/)[1].matchAll(/'([^']+)':/g)].map(match => match[1])
+  const api = read('docs/development-service-api.md')
+  const rows = part => [...part.matchAll(/^\| ([a-z][a-z.-]+) \|/gm)].map(match => match[1]).sort()
+  const queryText = api.split('## v1查询')[1].split('## v1命令')[0]
+  const commandText = api.split('## v1命令')[1].split('## Execution')[0]
+  assert.deepEqual(rows(queryText), [...queries].sort())
+  assert.deepEqual(rows(commandText), operations.filter(name => !queries.includes(name)).sort())
+  assert.equal(queries.length, 24); assert.equal(rows(commandText).length, 26)
+  for (const name of ['README.md', 'README.zh.md']) assert.match(read(name), /24查询\/26命令/)
+})
+
+test('quickstart JSON, shell syntax and local documentation links stay usable', () => {
+  for (const name of docs) {
+    const text = read(name)
+    for (const match of text.matchAll(/```json\n([\s\S]*?)\n```/g)) assert.doesNotThrow(() => JSON.parse(match[1]), `${name}: invalid JSON`)
+    for (const match of text.matchAll(/```bash\n([\s\S]*?)\n```/g)) execFileSync('bash', ['-n'], { input: match[1] })
+    for (const match of text.matchAll(/\]\(([^)]+)\)/g)) {
+      const target = match[1].split('#')[0]
+      if (!target || /^[a-z]+:/i.test(target)) continue
+      assert.ok(existsSync(resolve(dirname(resolve(root, name)), decodeURIComponent(target))), `${name}: missing ${target}`)
+    }
+  }
+  for (const name of ['README.md', 'README.zh.md']) {
+    const text = read(name)
+    assert.doesNotMatch(text, /1811e3b8f|fb_dev\(|fb_api\(|owner@flybot\.example|SuperAgent/)
+    assert.match(text, /node scripts\/backend\.mjs init/)
+    assert.match(text, /node scripts\/backend\.mjs start/)
+    assert.match(text, /README\.native/)
+  }
+})

.github/scripts/backend-runtime.test.mjs

@@ -0,0 +1,120 @@
+// Exercise the production launcher in an isolated persistent home. Inference is a test double.
+import assert from 'node:assert/strict'
+import { spawn } from 'node:child_process'
+import { once } from 'node:events'
+import { existsSync, readFileSync, writeFileSync, mkdtempSync, mkdirSync, readdirSync, rmSync, symlinkSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { createServer } from 'node:net'
+import { test } from 'node:test'
+import { backendConfig, callApi, connection, prepareBackend, readJson, root } from '../../scripts/backend-lib.mjs'
+import { mockModel } from '../../dev/demo/mock.mjs'
+import { agentReply } from '../../dev/demo/agents.mjs'
+
+const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
+async function freePort() { const server = createServer(); server.listen(0, '127.0.0.1'); await once(server, 'listening'); const port = server.address().port; await new Promise(resolve => server.close(resolve)); return port }
+async function run(args, env) {
+  const child = spawn(process.execPath, [join(root, 'scripts/backend.mjs'), ...args], { cwd: root, env, stdio: ['ignore', 'pipe', 'pipe'] })
+  let stdout = '', stderr = ''
+  child.stdout.on('data', bytes => { stdout += bytes }); child.stderr.on('data', bytes => { stderr += bytes })
+  const [code] = await once(child, 'exit')
+  if (code !== 0) throw new Error(`CLI refused: ${stderr}`)
+  return stdout
+}
+async function launch(home, env) {
+  const child = spawn(process.execPath, [join(root, 'scripts/backend.mjs'), 'start', '--home', home], { cwd: root, env, stdio: ['ignore', 'pipe', 'pipe'] })
+  let output = ''
+  child.stdout.on('data', bytes => { output += bytes }); child.stderr.on('data', bytes => { output += bytes })
+  const exited = once(child, 'exit')
+  const close = async () => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM'); await exited }
+  try {
+    for (let pass = 0; pass < 300; ++pass) {
+      if (output.includes('Backend ready:')) return { child, close, output: () => output }
+      if (child.exitCode !== null || child.signalCode !== null) throw new Error(`Production launcher failed: ${output}`)
+      await pause(100)
+    }
+    throw new Error('Production launcher readiness timed out')
+  } catch (error) { await close(); throw error }
+}
+
+test('production backend retains canonical Tasks/auth across restart, keeps consultation private and serves scoped SSE', { timeout: 90000 }, async t => {
+  const home = mkdtempSync(join(tmpdir(), 'flybot-backend-runtime-')), port = await freePort()
+  const model = mockModel([], { reply: agentReply })
+  model.listen(0, '127.0.0.1'); await once(model, 'listening')
+  const env = { ...process.env, DEEPSEEK_API_KEY: 'isolated-test-provider', DEEPSEEK_BASE_URL: `http://127.0.0.1:${model.address().port}/v1` }
+  const profile = prepareBackend(home, backendConfig({ port }))
+  // Offline test dependency provision; the separate fresh-home acceptance runs the real init installer.
+  const modules = join(profile, 'node_modules'), installed = join(root, 'core/dev/harness/node_modules')
+  mkdirSync(modules)
+  for (const entry of readdirSync(installed)) if (!entry.startsWith('.') && !['@flybot', 'ds-bot'].includes(entry)) symlinkSync(join(installed, entry), join(modules, entry))
+  symlinkSync(join(root, 'core'), join(modules, 'ds-bot'))
+  mkdirSync(join(modules, '@flybot'), { recursive: true })
+  for (const name of ['social', 'tree', 'timeline', 'workflow']) symlinkSync(join(root, 'plugins', name), join(modules, '@flybot', name))
+  let host
+  t.after(async () => { await host?.close(); model.closeAllConnections(); await new Promise(resolve => model.close(resolve)); rmSync(home, { recursive: true, force: true }) })
+  host = await launch(home, env)
+  await assert.rejects(run(['init', '--home', home], env), /already active/)
+  const login = await run(['login', '--home', home], env)
+  assert.match(login, /Authenticated/); assert.doesNotMatch(login + host.output(), /\?token=|sessionToken|isolated-test-provider/)
+  const token = readJson(join(home, 'private/backend-session.json')).sessionToken
+  const transport = await connection(home), api = (endpoint, body = {}, who = token) => callApi(transport, endpoint, body, who)
+  const dev = (kind, name, projectId, payload = {}, requestId) => api('project-development', { version: 'project-development/v1', kind, name, projectId, payload, ...(requestId ? { requestId } : {}) })
+  const { project } = await api('project-create', { name: 'Persistent service' })
+  const proposed = await dev('command', 'task.propose', project.id, { title: 'Project change', criteria: 'Result checked', impact: 'Keep project history' }, 'one-original-intent')
+  const taskId = proposed.result.id
+  assert.ok(taskId)
+  assert.equal((await dev('query', 'task', project.id, { taskId })).result.task.phase, 'awaiting-chief')
+  const inquiry = await dev('command', 'consultation.ask', project.id, { text: 'Explain the Project status without creating work' }, 'ordinary-question')
+  assert.equal(inquiry.result.consultation.initiativeTransferred, false)
+  let answer
+  for (let pass = 0; pass < 100; ++pass) {
+    answer = (await dev('query', 'consultation.get', project.id, { consultationId: inquiry.result.consultation.id })).result.consultation
+    if (answer.state === 'completed' && answer.replyCount > 0) break
+    await pause(100)
+  }
+  assert.equal(answer.state, 'completed')
+  assert.ok(answer.replyCount > 0)
+  assert.equal((await dev('query', 'tasks', project.id)).result.tasks.length, 1)
+  const external = await api('personal-agent-create', { name: 'External protocol caller', kind: 'external', provider: 'http-contract', endpoint: 'https://agent.example.invalid/callback' })
+  await api('personal-agent-join', { agentId: external.agent.id, projectId: project.id })
+  const credential = await api('personal-agent-token-create', { agentId: external.agent.id, projectIds: [project.id] })
+  const privateProposal = await api('personal-agent-propose-task', { projectId: project.id, title: 'Proposed by the independent agent', criteria: 'Owner reviews the proposal', impact: 'Retain explicit Project impact', scope: 'Transport mapping only', projectTaskKind: 'code', requestId: 'external-safe-proposal' }, credential.token)
+  assert.equal(privateProposal.proposal.fields.impact, 'Retain explicit Project impact')
+  assert.equal(privateProposal.proposal.fields.scope, 'Transport mapping only')
+  assert.equal((await dev('query', 'tasks', project.id)).result.tasks.length, 1)
+  const publication = await dev('command', 'proposal.publish', project.id, { proposalId: privateProposal.proposal.id }, 'human-approved-agent-proposal')
+  assert.equal(publication.result.task.projectTask.requirements.impact, 'Retain explicit Project impact')
+  await api('personal-agent-delegation-set', { agentId: external.agent.id, projectId: project.id, mode: 'watch' })
+  const watched = await api('project-development', { version: 'project-development/v1', kind: 'command', name: 'task.propose', projectId: project.id, requestId: 'external-watch-intent', payload: { title: 'Delegated Project work', criteria: 'Current authority checked', impact: 'Exercise the delegated publish boundary' } }, credential.token)
+  assert.ok(watched.result.id)
+  await api('personal-agent-token-revoke', { credentialId: credential.credential.id })
+  await assert.rejects(api('project-read', { id: project.id }, credential.token), /unauthorized|forbidden/)
+  const invitation = await api('auth-invite'), bob = await api('auth-redeem', { token: invitation.token, displayName: 'Bob' })
+  await assert.rejects(api('project-read', { id: project.id }, bob.sessionToken), /forbidden/)
+  const projectInvitation = await api('project-invite', { id: project.id })
+  await api('project-join', { token: projectInvitation.token }, bob.sessionToken)
+  await assert.rejects(api('project-development', { version: 'project-development/v1', kind: 'query', name: 'consultation.get', projectId: project.id, payload: { consultationId: inquiry.result.consultation.id } }, bob.sessionToken), /unavailable|forbidden|consultation/i)
+  const stream = await fetch(`${transport.baseUrl}/api/project-events?projectId=${project.id}`, { headers: { 'x-flybot-member-session': bob.sessionToken }, signal: AbortSignal.timeout(10000) })
+  assert.equal(stream.status, 200)
+  const reader = stream.body.getReader()
+  assert.match(new TextDecoder().decode((await reader.read()).value), /event: snapshot/)
+  await api('project-member-remove', { id: project.id, memberId: bob.memberId })
+  let revoked = ''
+  while (!revoked.includes('access-revoked')) { const next = await reader.read(); if (next.done) break; revoked += new TextDecoder().decode(next.value) }
+  assert.match(revoked, /access-revoked/)
+  await reader.cancel()
+  await host.close(); host = null
+  assert.equal(existsSync(join(home, 'bot/state.sqlite')), true)
+  host = await launch(home, env)
+  assert.match(await run(['login', '--home', home], env), /existing private local session/)
+  const nextTransport = await connection(home)
+  const after = await callApi(nextTransport, 'project-development', { version: 'project-development/v1', kind: 'command', name: 'task.propose', projectId: project.id, requestId: 'one-original-intent', payload: { title: 'Project change', criteria: 'Result checked', impact: 'Keep project history' } }, token)
+  assert.equal(after.replayed, true)
+  assert.equal(after.result.id, taskId)
+  await assert.rejects(callApi(nextTransport, 'project-read', { id: project.id }, bob.sessionToken), /forbidden/)
+  await host.close(); host = null
+  const statePath = join(home, 'bot/state.json'), future = JSON.stringify({ version: 99, note: 'Preserve this unsupported guard' })
+  writeFileSync(statePath, future)
+  await assert.rejects(launch(home, env), /read-only/)
+  assert.equal(readFileSync(statePath, 'utf8'), future)
+})

.github/scripts/backend.test.mjs

@@ -0,0 +1,56 @@
+import assert from 'node:assert/strict'
+import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'node:test'
+import { backendConfig, callApi, optionsOf, prepareBackend, privateJson, requestBody } from '../../scripts/backend-lib.mjs'
+import { createServer } from 'node:http'
+
+test('backend source profile preserves original plugins and uses real persistent pools rather than fixtures', t => {
+  const home = mkdtempSync(join(tmpdir(), 'flybot-backend-profile-'))
+  t.after(() => rmSync(home, { recursive: true, force: true }))
+  const config = backendConfig(), profile = prepareBackend(home, config)
+  const path = join(profile, 'package.json'), manifest = JSON.parse(readFileSync(path, 'utf8'))
+  for (const name of ['ds-bot', '@flybot/tree', '@flybot/workflow', '@flybot/social', '@flybot/timeline']) assert.ok(manifest.dsh.profile.bundles.includes(name))
+  assert.equal(manifest.dependencies['@deepseek-ai/dsh-base'], '0.2.1-rc.2')
+  assert.equal(manifest.dependencies['@deepseek-ai/dsh-web-app'], '0.2.1-rc.2')
+  assert.equal(config.bot.stateStorage, 'sqlite')
+  assert.equal(config.bot.projectWorkbench, true)
+  assert.equal(config.bot.projectDispatch, 'pools')
+  manifest.dependencies.custom = '1.0.0'; manifest.dsh.profile.bundles.push('custom'); privateJson(path, manifest)
+  prepareBackend(home, config)
+  assert.ok(JSON.parse(readFileSync(path, 'utf8')).dsh.profile.bundles.includes('custom'))
+  assert.equal(statSync(join(home, 'backend.json')).mode & 0o777, 0o600)
+  const overlay = readFileSync(join(home, 'backend.patch.yml'), 'utf8')
+  assert.match(overlay, /projectWorkbench/)
+  assert.doesNotMatch(overlay, /fixture|mock|API_KEY|sessionToken/)
+})
+
+test('backend rejects ambiguous configuration and malformed CLI command identity', () => {
+  for (const config of [{ port: 0 }, { port: '3080' }, { unknown: true }, { bot: [] }, { bot: { projectWorkbench: false } }, { bot: { projectDispatch: 'workflow' } }, { bot: { home: '/somewhere' } }]) assert.throws(() => backendConfig(config))
+  assert.deepEqual(optionsOf(['query', 'tasks', '--project', 'p', '--home', '/tmp/user-home']), { command: 'query', positional: ['tasks'], options: { project: 'p', home: '/tmp/user-home' } })
+  for (const args of [['start', '--home'], ['start', '--password', 'secret'], ['start', '--home', 'a', '--home', 'b']]) assert.throws(() => optionsOf(args))
+  assert.throws(() => requestBody('command', 'task.propose', { project: 'p' }), /request-id/)
+  assert.throws(() => requestBody('query', 'tasks', { project: 'p', json: '[]' }), /object/)
+  const request = requestBody('command', 'task.propose', { project: 'p', 'request-id': 'one-intent', json: '{"title":"Project change"}' })
+  assert.equal(request.endpoint, 'project-development')
+  assert.equal(request.payload.version, 'project-development/v1')
+  assert.equal(request.payload.requestId, 'one-intent')
+  assert.equal(request.payload.payload.title, 'Project change')
+})
+
+test('CLI HTTP client uses existing authenticated API and reports domain/transport failures', async t => {
+  let seen
+  const server = createServer(async (request, response) => {
+    let text = ''; for await (const bytes of request) text += bytes
+    seen = { url: request.url, cookie: request.headers.cookie, member: request.headers['x-flybot-member-session'], request: JSON.parse(text) }
+    response.setHeader('content-type', 'application/json')
+    response.end(JSON.stringify(seen.request.endpoint === 'deny' ? { ok: false, error: { code: 'bot/forbidden', message: 'Current membership required' } } : { ok: true, value: { projectId: 'p' } }))
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  t.after(() => new Promise(resolve => server.close(resolve)))
+  const transport = { baseUrl: `http://127.0.0.1:${server.address().port}`, cookie: 'transport=private' }
+  assert.deepEqual(await callApi(transport, 'project-read', { id: 'p' }, 'member-secret'), { projectId: 'p' })
+  assert.equal(seen.url, '/api/bot'); assert.equal(seen.member, 'member-secret'); assert.equal(seen.cookie, 'transport=private')
+  await assert.rejects(callApi(transport, 'deny', {}, 'member-secret'), /bot\/forbidden/)
+})

LESSONS.md

@@ -771,3 +771,7 @@ Seen 1× (actual SQLite API regression). A Task query requested full delivery de
 ### 2026-11-03 — Absence of a new contract is not legacy admission
 
 Seen 1× (group dispatch and Bot tracking reproduced the bypass). Capture existing Project identities once with committed evidence; later untyped rows must not inherit admission. Preserve original rework and explicit Workflow execution through trusted continuation/controller ports. Scope guards must run before mutating leases or claims, and physical storage rollback must retain the code-version fence.
+
+### 2026-11-03 — Validate the composed external proposal, not just its private receipt
+
+Seen 1× (production-entry FBA regression). Legacy external proposals preserved title/criteria but dropped canonical impact/scope, leaving the approved action unable to carry its original requirements to Chief. Preserve explicit fields in both the private intent hash and published original Task; test actual authenticated HTTP approval/watch/revoke. Successful startup also requires actual Store writability, not only a Harness login URL.

PROJECT.md

@@ -288,3 +288,7 @@ Service composition adds scoped native execution/exit and current-vs-pinned role
 ### 2026-11-03 — Preserve original capabilities (#2186)
 
 Keep personal Chief/Bots, groups, private memories and original Task/Workflow IDs. Shared Project legacy execution requires an identity-bound committed cutover; new dispatch/tracking cannot bypass Chief admission. Associated original groups retain their own history ACL plus current Project access. JSON version4 fences new contracts across SQLite rollback; code downgrades must not erase receipts. No deployment or UI replacement. See docs/subsystems/project-compatibility.md.
+
+### 2026-11-03 — Persistent deployment and final API docs (#2189)
+
+Use scripts/backend.mjs to initialize/start the original DSH/FlyBot host plus existing restricted HTTP/SSE gateway in a private persistent home. Login and CLI calls reuse native/platform identity; no mock model, sample repository, parallel executor or frontend replacement is packaged. Original guides move to README.native variants. Auth-mode confirms Store writability before readiness. External safe proposal impact/scope must survive publication into canonical requirements (actual production-entry regression). Deployment/restart, deterministic HTTP/DSH tests and real-provider acceptance remain separate evidence.