host: the state journal says it has not started until the first save (#2373)

changelog.d/2373-journal-not-started.fixed.md

@@ -0,0 +1 @@
+The diagnostics report no longer says the state journal is `recording` before this host has saved the team: a journal whose writer has not opened yet now reads `state journal: not started (opens with the first save)`, so a window that never saves stops promising lines it never asked the file for ([#2373](https://github.com/ThinkFlowLab/FlyBot/issues/2373)).

core/src/host/diagnostics.js

@@ -237,12 +237,14 @@ export function install(rt) {
       // live state.lock are different faults, and the sidebar shows this same sentence, so the
       // report must not compress both into "another dsh holds the team".
       readOnly: readOnly ? clean(cut(readOnly, 200).text) : undefined,
-      // The writer is opened before the lock is asked for, so in a window another dsh keeps
-      // read-only `save()` returns before `journal.record()` and no line is written at all;
-      // claiming `recording` there promises a line the file will not get. A journal that
+      // The writer opens on the first save that got past `writable()` (#2351), so a window that
+      // has not saved yet never read or repaired the file: `recording` there promises a line
+      // nobody asked for, and `opened: false` says so instead. A window another dsh keeps
+      // read-only says that, because it never reaches that first save either. A journal that
       // genuinely stopped keeps its own reason, which is the fault a report is usually for.
       stateJournal: journal === undefined ? undefined
-        : journal.recording ? (readOnly ? 'paused (this window is read-only)' : 'recording')
+        : journal.recording ? (readOnly ? 'paused (this window is read-only)'
+          : journal.opened === false ? 'not started (opens with the first save)' : 'recording')
           : `stopped (${clean(journal.because)})`,
     }
   }

core/src/host/store.js

@@ -55,7 +55,10 @@ export function install(rt) {
     ? {
         path: journalPath,
         record: (snapshot) => (journalWriter ??= openJournal({ path: journalPath, warn })).record(snapshot),
-        status: () => journalWriter?.status() ?? { recording: true },
+        // Until that first save there is no writer to answer: `opened: false` says the journal was
+        // never read, so `recording` cannot promise a line this host never asked for. Reading the
+        // status opens nothing — opening here would be the write the lazy open avoids (#2351).
+        status: () => (journalWriter === undefined ? { recording: true, opened: false } : { ...journalWriter.status(), opened: true }),
       }
     : undefined
 

core/test/diagnostics.test.js

@@ -327,6 +327,29 @@ test('a read-only window does not say the state journal is recording (#2153)', a
   assert.match(stoppedHeld.markdown, /^state journal: stopped \(.*not a plain file.*\)$/m)
 })
 
+// The writer opens on the first save, so a host that has not saved yet has not read or repaired
+// anything: `recording` there promises a line nobody asked the file for. A journal with no
+// `opened` is the shape before this change and still says `recording`.
+test('a state journal that has not opened says it starts with the first save (#2373)', async (t) => {
+  const home = mkdtempSync(join(tmpdir(), 'ds-bot-diag-'))
+  t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))
+  const notStarted = { status: () => ({ recording: true, opened: false }) }
+  const fresh = await host(home, { journal: notStarted, config: { diagnosticsFile: false } }).diagnostics.report({})
+  assert.match(fresh.markdown, /^state journal: not started \(opens with the first save\)$/m)
+  assert.equal(fresh.facts.stateJournal, 'not started (opens with the first save)')
+
+  const legacy = { status: () => ({ recording: true }) }
+  const old = await host(home, { journal: legacy, config: { diagnosticsFile: false } }).diagnostics.report({})
+  assert.match(old.markdown, /^state journal: recording$/m)
+  assert.equal(old.facts.stateJournal, 'recording')
+
+  // A read-only window keeps its own wording, before the first save as after it.
+  const readOnly = () => 'Another dsh (pid 1 on host) holds the team'
+  const held = await host(home, { journal: notStarted, readOnly, config: { diagnosticsFile: false } }).diagnostics.report({})
+  assert.match(held.markdown, /^state journal: paused \(this window is read-only\)$/m)
+  assert.equal(held.facts.stateJournal, 'paused (this window is read-only)')
+})
+
 // The Read-only row answered one constant for every window that may not write, but the store
 // answers three ways (#2153): `stuck` (the team file cannot be read) is not a live lock at all,
 // and a report that calls a newer or unreadable team file "another dsh holds the team" sends the

core/test/store.test.js

@@ -286,6 +286,54 @@ test('a read-only window neither creates nor rewrites the state journal', async
   assert.deepEqual(fs.readFileSync(path), before, 'a refused save leaves the journal bytes alone')
 })
 
+// A home that already holds a saved team, so loading a fresh host is only a read: with no
+// state.json, load() saves the file itself and that first save would open the writer.
+async function savedHome(t) {
+  const home = fs.mkdtempSync(join(tmpdir(), 'ds-bot-store-'))
+  t.after(() => fs.rmSync(home, { recursive: true, force: true, maxRetries: 5 }))
+  const first = runtime(t, undefined, { home })
+  await first.rt.load()
+  await first.rt.save()
+  await first.dispose()
+  return home
+}
+
+// The writer opens on the first save, so until then nothing was read, folded or repaired:
+// `recording: true` alone promises lines the file was never asked for. `opened` says which of
+// the two states a reader is looking at, and reading it never opens the writer — that is the
+// write the lazy open exists to avoid.
+test('the state journal says it has not started until the first save opens it (#2373)', async (t) => {
+  const home = await savedHome(t)
+  const { rt, warnings } = runtime(t, undefined, { home, config: { stateJournal: true } })
+  assert.deepEqual(rt.journal.status(), { recording: true, opened: false })
+  await rt.load()
+  assert.deepEqual(rt.journal.status(), { recording: true, opened: false }, 'loading a saved team opens nothing')
+  assert.equal(fs.existsSync(join(home, 'journal')), false, 'a host that never saved creates no journal folder')
+  await rt.save()
+  assert.deepEqual(rt.journal.status(), { recording: true, opened: true }, 'the first save opens the writer')
+  assert.equal(fs.existsSync(join(home, 'journal', 'state.jsonl')), true)
+  assert.equal(warnings.length, 0, 'a journal that opened cleanly warns about nothing')
+})
+
+// A journal that refuses is only found out by the first save, which is the first read of the
+// file: before it, the honest answer is still "not started", not a stop that never happened.
+test('a journal that refuses on the first save has only then stopped (#2373)', async (t) => {
+  const home = await savedHome(t)
+  const target = join(home, 'neighbour.txt')
+  fs.writeFileSync(target, 'PRIVATE LINE\n')
+  fs.mkdirSync(join(home, 'journal'), { recursive: true })
+  fs.symlinkSync(target, join(home, 'journal', 'state.jsonl'))
+  const { rt, warnings } = runtime(t, undefined, { home, config: { stateJournal: true } })
+  await rt.load()
+  assert.deepEqual(rt.journal.status(), { recording: true, opened: false }, 'loading the host reads no journal')
+  await rt.save()
+  const status = rt.journal.status()
+  assert.equal(status.opened, true, 'the first save opens the writer')
+  assert.equal(status.recording, false, 'a journal that refuses does not record')
+  assert.match(status.because, /not a plain file/)
+  assert.equal(warnings.length, 1, 'a refused start-up is warned once')
+})
+
 test('dispose waits for a pending save and prevents later writes', async (t) => {
   const { rt, home, statePath, dispose } = runtime(t)
   await rt.load()

docs/subsystems/core-host.md

@@ -558,7 +558,7 @@ A local, replayable record of every change to the saved team, for multi-Bot bugs
 - **On.** `stateJournal: true` in the `bot` config. Off (the default) nothing is opened or written, so every other test runs as before.
 - **What.** `src/host/journal.js`. After each successful `state.json` write, `store.js` hands the exact snapshot text to the journal, which appends the difference from the last recorded snapshot to `<home>/journal/state.jsonl` (file mode 0600, folder 0700). Lines are `{ seq, t, op, key, prev, next, botId?, actor }`; `op` is `base` (the whole team: the first line, and any change only a key reorder explains), `set`, `delete`, or `append` (`prevLength`, `drop`, the new items: a group log or other list that grew or slid). Paths go at most three keys deep (`bots.<id>.<field>`); `botId` is set on a `bots.<id>` path; `actor` is `host` for a save and `retention` for a folded base (per-call attribution is not recorded). Only the host that holds the state lock writes, so a read-only window records nothing. Opening the writer is itself a write — it creates `journal/` (mode 0700) and repairs a half-written last line by replacing the file with its prefix — so `store.js` opens it on the first save that got past `writable()`, not when the host starts: a window another dsh keeps read-only creates no folder and leaves the holder's lines as they are, and the report still says `paused (this window is read-only)`.
 - **Retention.** When a write would pass 20 MB, or the oldest change is older than 7 days, the oldest lines are dropped down to 90% of each cap and folded into the base line (atomic rewrite), so the kept lines still replay. A team that alone exceeds the cap stops recording.
-- **Fail closed.** Replay (`replayTo(text, seq)`) refuses the whole journal on an unreadable or incomplete line, an unknown op, a gap in `seq`, a first line that is not a base, or a `prev` that does not match what the earlier lines built; a `seq` outside the file is refused, not approximated. On start the writer replays the existing file the same way; a journal that does not replay, a path that is not a plain file, or a failed write stops recording with a warning and leaves the file as it is. A last line left half written by a crash is the one shape the writer repairs instead of refusing: when the complete prefix before it replays, the file is replaced with that prefix (the same atomic temp-and-rename as a rotation) with one warning, `dropped an incomplete last line left by a crash`, and recording continues from it ([#2166](https://github.com/ThinkFlowLab/FlyBot/issues/2166)) — a broken line above the last one, a `seq` gap, a `prev` mismatch or an unknown op still stops recording and leaves the file as it is. A writer that stopped is asked once more on the next save: it runs the start-up `open()` again, so a folder or file repaired meanwhile resumes recording — appending the change that save carried, or starting a new file with a base when the journal was removed — while a journal that still refuses stays stopped and that save still lands; the asking is silent, so a refused start-up or write is warned once, not once per save ([#2154](https://github.com/ThinkFlowLab/FlyBot/issues/2154)). Saving the team never depends on the journal. The folder above the journal must be a real directory too (#2056): a link, a plain file or a check that cannot run refuses before the `mkdir`/`chmod` that would follow it, so neither the permissions nor a write reach the folder a link names; a folder that does not exist yet is still this writer's to create, as before. Since [#2074](https://github.com/ThinkFlowLab/FlyBot/issues/2074) the same question is asked again when the write runs: a journal folder swapped for a link is refused with one warning and recording stops, and a `state.jsonl` swapped for a link is never written through. A hard link reads as a plain file to `lstat` while its bytes are also a name outside this home, so the start-up read, the append and — since [#2146](https://github.com/ThinkFlowLab/FlyBot/issues/2146) — the rotation also require the file to carry only this one name (`nlink === 1`): a shared inode is refused exactly like a link — one warning, nothing read, chmod-ed or appended — and a journal with one name records as before. The append path refuses the write, and so does the rotation, before it reads: a link or a second name there stops recording with one warning, so another file's lines are never read as the team's history and the `rename` never drops the other name. A writer that stopped says so off the host too: `rt.journal.status()` answers `{ recording, because }`, and the diagnostics report adds one line — `state journal: recording`, `state journal: stopped (<reason>)`, or `state journal: paused (this window is read-only)` in a window another dsh keeps read-only, which never reaches the write that records ([#2153](https://github.com/ThinkFlowLab/FlyBot/issues/2153)) — while a host without the journal adds none ([#2134](https://github.com/ThinkFlowLab/FlyBot/issues/2134)).
+- **Fail closed.** Replay (`replayTo(text, seq)`) refuses the whole journal on an unreadable or incomplete line, an unknown op, a gap in `seq`, a first line that is not a base, or a `prev` that does not match what the earlier lines built; a `seq` outside the file is refused, not approximated. On the first save the writer replays the existing file the same way; a journal that does not replay, a path that is not a plain file, or a failed write stops recording with a warning and leaves the file as it is. A last line left half written by a crash is the one shape the writer repairs instead of refusing: when the complete prefix before it replays, the file is replaced with that prefix (the same atomic temp-and-rename as a rotation) with one warning, `dropped an incomplete last line left by a crash`, and recording continues from it ([#2166](https://github.com/ThinkFlowLab/FlyBot/issues/2166)) — a broken line above the last one, a `seq` gap, a `prev` mismatch or an unknown op still stops recording and leaves the file as it is. A writer that stopped is asked once more on the next save: it runs the start-up `open()` again, so a folder or file repaired meanwhile resumes recording — appending the change that save carried, or starting a new file with a base when the journal was removed — while a journal that still refuses stays stopped and that save still lands; the asking is silent, so a refused start-up or write is warned once, not once per save ([#2154](https://github.com/ThinkFlowLab/FlyBot/issues/2154)). Saving the team never depends on the journal. The folder above the journal must be a real directory too (#2056): a link, a plain file or a check that cannot run refuses before the `mkdir`/`chmod` that would follow it, so neither the permissions nor a write reach the folder a link names; a folder that does not exist yet is still this writer's to create, as before. Since [#2074](https://github.com/ThinkFlowLab/FlyBot/issues/2074) the same question is asked again when the write runs: a journal folder swapped for a link is refused with one warning and recording stops, and a `state.jsonl` swapped for a link is never written through. A hard link reads as a plain file to `lstat` while its bytes are also a name outside this home, so the start-up read, the append and — since [#2146](https://github.com/ThinkFlowLab/FlyBot/issues/2146) — the rotation also require the file to carry only this one name (`nlink === 1`): a shared inode is refused exactly like a link — one warning, nothing read, chmod-ed or appended — and a journal with one name records as before. The append path refuses the write, and so does the rotation, before it reads: a link or a second name there stops recording with one warning, so another file's lines are never read as the team's history and the `rename` never drops the other name. A writer that stopped says so off the host too: `rt.journal.status()` answers `{ recording, opened, because? }` — `opened: false` while no save has opened the writer yet, and `opened` never claimed otherwise — and the diagnostics report adds one line — `state journal: recording`, `state journal: not started (opens with the first save)` while the writer has not opened, `state journal: stopped (<reason>)`, or `state journal: paused (this window is read-only)` in a window another dsh keeps read-only, which never reaches the write that records ([#2153](https://github.com/ThinkFlowLab/FlyBot/issues/2153)) — while a host without the journal adds none ([#2134](https://github.com/ThinkFlowLab/FlyBot/issues/2134)).
 - **Replay.** `core/scripts/replay.mjs` (dev-only, not in the package, see [install](install.md)) prints or writes (`--out`, new file only, 0600) the team after a given `--to <seq>`; it makes no network or process calls. Test: `core/test/state-journal.test.js` (a 3-Bot round replays byte-identical at every step).
 
 ### Standing tool approvals (#540)

patches/ds-bot/0001-core-fixes.patch

@@ -5085,7 +5085,7 @@ diff --git a/src/host/diagnostics.js b/src/host/diagnostics.js
      return {
        botVersion: ownVersion(),
        dshVersion: dshVersion(),
-@@ -199,12 +199,22 @@ export function install(rt) {
+@@ -199,12 +199,24 @@ export function install(rt) {
        node: process.version,
        team: `${count(bots.length, 'Bot', 'Bots')}, ${rt.state.mainBotIds.length} Main, ${count(groups, 'group chat', 'group chats')}`,
        models: models.length > 0 ? models.join(', ') : undefined,
@@ -5094,12 +5094,14 @@ diff --git a/src/host/diagnostics.js b/src/host/diagnostics.js
 +      // live state.lock are different faults, and the sidebar shows this same sentence, so the
 +      // report must not compress both into "another dsh holds the team".
 +      readOnly: readOnly ? clean(cut(readOnly, 200).text) : undefined,
-+      // The writer is opened before the lock is asked for, so in a window another dsh keeps
-+      // read-only `save()` returns before `journal.record()` and no line is written at all;
-+      // claiming `recording` there promises a line the file will not get. A journal that
++      // The writer opens on the first save that got past `writable()` (#2351), so a window that
++      // has not saved yet never read or repaired the file: `recording` there promises a line
++      // nobody asked for, and `opened: false` says so instead. A window another dsh keeps
++      // read-only says that, because it never reaches that first save either. A journal that
 +      // genuinely stopped keeps its own reason, which is the fault a report is usually for.
 +      stateJournal: journal === undefined ? undefined
-+        : journal.recording ? (readOnly ? 'paused (this window is read-only)' : 'recording')
++        : journal.recording ? (readOnly ? 'paused (this window is read-only)'
++          : journal.opened === false ? 'not started (opens with the first save)' : 'recording')
 +          : `stopped (${clean(journal.because)})`,
      }
    }

patches/ds-bot/0002-flybot-api-host.patch

@@ -784,7 +784,7 @@ diff --git a/src/host/roster.js b/src/host/roster.js
 diff --git a/src/host/store.js b/src/host/store.js
 --- a/src/host/store.js
 +++ b/src/host/store.js
-@@ -1,21 +1,63 @@
+@@ -1,21 +1,66 @@
  // Store: the saved team (state.json), its migrations, the lock that keeps a second dsh
  // host on the same data from writing it, and the indexes that map a Session to the Bot
  // or group chat it belongs to.
@@ -848,7 +848,10 @@ diff --git a/src/host/store.js b/src/host/store.js
 +    ? {
 +        path: journalPath,
 +        record: (snapshot) => (journalWriter ??= openJournal({ path: journalPath, warn })).record(snapshot),
-+        status: () => journalWriter?.status() ?? { recording: true },
++        // Until that first save there is no writer to answer: `opened: false` says the journal was
++        // never read, so `recording` cannot promise a line this host never asked for. Reading the
++        // status opens nothing — opening here would be the write the lazy open avoids (#2351).
++        status: () => (journalWriter === undefined ? { recording: true, opened: false } : { ...journalWriter.status(), opened: true }),
 +      }
 +    : undefined