transcript.mjs

#!/usr/bin/env node
// Summarise and audit one evaluator's session.
//
//     node eval/transcript.mjs <case.jsonl> [--calls] [--report]
//
// eval/run_case.mjs writes the session as Claude Code's stream-json output and
// prints this digest at the end of every run. See eval/README.md.
//
// WHY THE AUDIT EXISTS
//
// An evaluator's account of its own channels is not reliable. An evaluator can
// open with a full-text search of the whole corpus for the gate's name, before
// a single site search or navigation hop, and then report that Channel 3 was
// "not needed"; or find the answer by search and walk the navigation path
// afterwards, to confirm links it already knew. Such hops are real links, but
// they were not found blind, and both reports read as clean.
//
// A subagent hands back only its final report, so it cannot show either. The
// session records every call in order, and the order is what the protocol's
// channels are about. The audit reads it back; it does not score anything.

import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { exitOnCrash, parseCli, printHelpAndExit, withUsageError } from "../lib/cli.mjs";

/** Every event in a stream-json session, in order. */
export function readTranscript(file) {
  const events = [];
  for (const line of fs.readFileSync(file, "utf8").split("\n")) {
    if (!line.trim()) continue;
    try {
      events.push(JSON.parse(line));
    } catch {
      /* not an event */
    }
  }
  return events;
}

/**
 * The session's setup, its tool calls paired with their results, everything
 * the model wrote, and the final report.
 */
export function summarize(events) {
  const s = { init: null, result: null, calls: [], texts: [], report: "" };
  const byId = new Map();
  for (const e of events) {
    if (e.type === "system" && e.subtype === "init") s.init = e;
    else if (e.type === "result") s.result = e;
    else if (e.type === "assistant") {
      for (const c of e.message?.content ?? []) {
        if (c.type === "text" && c.text.trim()) {
          s.texts.push(c.text);
          s.report = c.text;
        }
        if (c.type !== "tool_use") continue;
        const call = { n: s.calls.length + 1, tool: c.name, input: c.input ?? {}, ok: null, output: "" };
        s.calls.push(call);
        byId.set(c.id, call);
      }
    } else if (e.type === "user" && Array.isArray(e.message?.content)) {
      for (const c of e.message.content) {
        const call = c.type === "tool_result" ? byId.get(c.tool_use_id) : undefined;
        if (!call) continue;
        call.ok = !c.is_error;
        call.output = typeof c.content === "string" ? c.content : JSON.stringify(c.content);
      }
    }
  }
  if (typeof s.result?.result === "string" && s.result.result.trim()) s.report = s.result.result;
  return s;
}

const SHELL_SEARCH = /(^|[\s;&|(])(grep|rg|findstr|select-string)\b/i;
const RECURSIVE = /\s-[a-zA-Z]*[rR]|--recursive/;

/**
 * The search box in command position, by name or by a path to the shim. A bare
 * substring match would count `which site-search; type site-search`, two
 * lookups of the shim, as two queries.
 */
const SEARCH_CALL = /(?:^|[;&|(]\s*)"?(?:[^\s;&|()"]*[\\/])?site-search\b/g;

/**
 * What a call was, in the protocol's terms. A search for a permalink is the
 * sanctioned way to open a search result, and a search inside one page is a
 * reader's find-in-page, so neither is full-text search. Any other search is,
 * whether or not it was refused. (Read-only commands inside the working
 * directory are not refused, so a shell grep can run.)
 */
export function classify(call) {
  const cmd = String(call.input.command ?? "");
  switch (call.tool) {
    case "Bash":
      if (queriesIn(call)) return "search";
      if (!SHELL_SEARCH.test(cmd)) return "shell";
      if (/permalink/i.test(cmd)) return "permalink";
      return !RECURSIVE.test(cmd) && /\.md\b/i.test(cmd) ? "find" : "fulltext";
    case "Grep":
      if (/permalink/i.test(String(call.input.pattern ?? ""))) return "permalink";
      return /\.md$/i.test(String(call.input.path ?? "")) ? "find" : "fulltext";
    case "Read":
      return "read";
    case "Glob":
      return "list";
    default:
      return "other";
  }
}

const LETTER = {
  search: "S",
  fulltext: "F",
  permalink: "P",
  find: "I",
  read: "R",
  list: "L",
  shell: "X",
  other: "O",
};

/** True when a line of the report says Channel 3 went unused. */
function claimsChannel3Unused(report) {
  for (const line of report.split("\n")) {
    if (!/channel 3/i.test(line)) continue;
    if (/\b(not needed|not used|not required|unnecessary|none|skipped|no)\b/i.test(line)) return true;
  }
  return false;
}

/**
 * How many queries one search call ran. An evaluator can chain several in one
 * command, and a timeline of one letter per call then reads as one search.
 */
function queriesIn(call) {
  return call.tool === "Bash" ? (String(call.input.command ?? "").match(SEARCH_CALL) ?? []).length : 0;
}

/** The facts about channel order the report cannot be trusted to state. */
export function audit(s) {
  const kinds = s.calls.map(classify);
  const firstSearch = s.calls.findIndex((c, i) => kinds[i] === "search" && c.ok !== false);
  const fulltext = s.calls.filter((_, i) => kinds[i] === "fulltext");
  const early = fulltext.filter((c) => firstSearch < 0 || c.n - 1 < firstSearch);
  // A refused call ran no query, so it is counted apart: a total that
  // included refused queries would overstate the searches made.
  const allSearchCalls = s.calls.filter((_, i) => kinds[i] === "search");
  const searchCalls = allSearchCalls.filter((c) => c.ok !== false);
  const refusedSearches = allSearchCalls.length - searchCalls.length;
  const queries = searchCalls.reduce((n, c) => n + queriesIn(c), 0);
  const flags = [];
  if (firstSearch < 0) flags.push("no site search at all: Channel 1 was not run");
  if (early.length) flags.push(`${early.length} full-text search(es) before the first site search`);
  if (fulltext.length && claimsChannel3Unused(s.report)) {
    flags.push(`the report says Channel 3 was not used, and the session has ${fulltext.length} full-text search(es)`);
  }
  const timeline = kinds
    .map((k, i) => {
      const letter = s.calls[i].ok === false ? LETTER[k].toLowerCase() : LETTER[k];
      const n = k === "search" ? queriesIn(s.calls[i]) : 1;
      return n > 1 ? `${letter}${n}` : letter;
    })
    .join(" ");
  return {
    kinds,
    firstSearch: firstSearch < 0 ? null : firstSearch + 1,
    fulltext,
    early,
    flags,
    timeline,
    queries,
    searchCalls: searchCalls.length,
    refusedSearches,
  };
}

const brief = (call) => {
  const i = call.input;
  const what = i.command ?? i.file_path ?? i.pattern ?? JSON.stringify(i);
  return `${call.tool} ${String(what).replace(/\s+/g, " ")}${i.path ? `  [in ${i.path}]` : ""}`;
};

/** The digest run_case.mjs prints after every run. */
export function printDigest(s, { calls = false, report = false } = {}) {
  const a = audit(s);
  const i = s.init ?? {};
  const r = s.result ?? {};
  const refused = s.calls.filter((c) => c.ok === false).length;
  console.log(
    `model ${i.model ?? "?"} -- Claude Code ${i.claude_code_version ?? "?"} -- ${i.permissionMode ?? "?"} -- tools ${(i.tools ?? []).join(",")}`,
  );
  console.log(`cwd   ${i.cwd ?? "?"}`);
  console.log(
    `${r.num_turns ?? "?"} turns -- ${Math.round((r.duration_ms ?? 0) / 1000)} s -- ` +
      `$${(r.total_cost_usd ?? 0).toFixed(3)} -- ${s.calls.length} calls, ${refused} refused or failed`,
  );
  console.log(`\ntimeline  ${a.timeline || "(no calls)"}`);
  console.log("          S site search  F full-text search  P permalink lookup  I find in one page");
  console.log("          R read  L listing  X other shell  (lower case: refused or failed)");
  console.log("          a number after a letter: that many queries in the one call");
  console.log(`\nfirst site search: ${a.firstSearch ? `call ${a.firstSearch}` : "none"}`);
  console.log(
    `site searches: ${a.queries} quer${a.queries === 1 ? "y" : "ies"} in ${a.searchCalls} call(s)` +
      (a.refusedSearches ? `, and ${a.refusedSearches} refused call(s) that ran none` : ""),
  );
  console.log(`full-text searches: ${a.fulltext.length}`);
  for (const c of a.fulltext) console.log(`  #${c.n} ${brief(c).slice(0, 160)}`);
  for (const f of a.flags) console.log(`FLAG  ${f}`);
  if (calls) {
    console.log("\ncalls");
    for (const c of s.calls) {
      console.log(`  #${String(c.n).padEnd(3)} ${c.ok === false ? "REFUSED " : ""}${brief(c).slice(0, 200)}`);
    }
  }
  if (report) console.log(`\n${s.report}`);
  return a;
}

const USAGE =
  "Usage: node eval/transcript.mjs <case.jsonl> [--calls] [--report] [-h, --help]\n\n" +
  "Summarises an evaluator's session and audits the order of its channels.\n" +
  "A file name that starts with a dash goes after --. See eval/README.md.\n\n" +
  "Exit codes:\n" +
  "  0  the digest was printed\n" +
  "  2  a refused command line, a session file that cannot be read, or a crash";

function main(argv) {
  const { values, positionals } = withUsageError(() =>
    parseCli(argv, {
      options: {
        calls: { type: "boolean", default: false },
        report: { type: "boolean", default: false },
        help: { type: "boolean", short: "h" },
      },
      positionals: { min: 0, max: 1 },
      stopAt: ["help"],
    }),
  );
  if (values.help) printHelpAndExit(USAGE);
  const [file] = positionals;
  if (!file) printHelpAndExit(USAGE, { stream: "stderr", exitCode: 2 });
  printDigest(summarize(readTranscript(file)), { calls: values.calls, report: values.report });
}

if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  exitOnCrash();
  main(process.argv.slice(2));
}