From 9c0493c4b07d44c78ea02be116b5cf7eb0beba05 Mon Sep 17 00:00:00 2001 From: GDS K S Date: Sun, 8 Nov 2026 20:50:12 -0600 Subject: [PATCH 1/5] feat: quiet repo mode and a trust line on impact --- internal/config/config.go | 5 +++++ internal/config/scan.go | 18 ++++++++++++++++++ internal/graph/impact.go | 21 +++++++++++++++++++++ internal/graph/view_test.go | 2 +- internal/setup/digest.go | 11 ++++++++++- internal/setup/session.go | 6 ++++++ internal/setup/small_test.go | 24 ++++++++++++++++++++++++ 7 files changed, 85 insertions(+), 2 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 24ed8fa..ffa10db 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,6 +44,11 @@ type Config struct { // digest. Nil uses DefaultSmallRepoFiles; 0 turns small repo mode off. // Also STACKLIT_SMALL_REPO_FILES. SmallRepoFiles *int `json:"small_repo_files,omitempty"` + // QuietRepoFiles is the source file count below which the session and + // subagent hooks inject nothing, because measured sessions cost more with + // them on repos this size. Nil uses DefaultQuietRepoFiles; 0 turns quiet + // mode off. Also STACKLIT_QUIET_REPO_FILES. + QuietRepoFiles *int `json:"quiet_repo_files,omitempty"` // CrawlGuard sets the Claude Code crawl guard hook mode: "off", "nudge" or // "redirect". Also STACKLIT_CRAWL_GUARD. Empty uses the mode setup installed. CrawlGuard string `json:"crawl_guard,omitempty"` diff --git a/internal/config/scan.go b/internal/config/scan.go index f56d18b..89fc054 100644 --- a/internal/config/scan.go +++ b/internal/config/scan.go @@ -31,6 +31,19 @@ const ( // STACKLIT_SMALL_REPO_FILES; 0 turns small repo mode off. const DefaultSmallRepoFiles = 600 +// DefaultQuietRepoFiles is the source file count under which the hooks stay +// silent. The agent test (bench/agent-ab/RESULTS-round2.md) lost money at +// 412 to 2,919 files and won at 15,978; the break-even between is unmeasured. +const DefaultQuietRepoFiles = 5000 + +// QuietRepoLimit returns the effective quiet threshold (0 means off). +func (c *Config) QuietRepoLimit() int { + if c.QuietRepoFiles == nil { + return DefaultQuietRepoFiles + } + return *c.QuietRepoFiles +} + // SmallRepoLimit returns the effective small repo threshold (0 means off). func (c *Config) SmallRepoLimit() int { if c.SmallRepoFiles == nil { @@ -67,6 +80,11 @@ func (c *Config) applyEnv() { c.SmallRepoFiles = &n } } + if v := os.Getenv("STACKLIT_QUIET_REPO_FILES"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + c.QuietRepoFiles = &n + } + } if v := os.Getenv("STACKLIT_WORKERS"); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { c.Workers = n diff --git a/internal/graph/impact.go b/internal/graph/impact.go index f892b73..21352cc 100644 --- a/internal/graph/impact.go +++ b/internal/graph/impact.go @@ -84,6 +84,9 @@ func (g *SymGraph) Impact(target string, maxDepth, budget int, asJSON bool) (str items[i] = item{node: v, tags: []string{fmt.Sprintf("depth %d %s", depth[v], impactConf(score[v]))}} } head := []string{fmt.Sprintf("%d dependents (%d direct) across %d files, depth<=%d", len(ids), direct, len(files), maxDepth)} + if t := trustLine(ids, score); t != "" { + head = append(head, t) + } hint := fmt.Sprintf("stacklit impact %s --budget %d", target, budget*2) return g.FitView(viewSpec{title: "impact " + g.Label(root), head: head, items: items, hint: hint}, budget, asJSON), nil } @@ -94,3 +97,21 @@ func impactConf(s float64) string { } return fmt.Sprintf("likely %.2f", s) } + +// trustLine tells the agent how much of the list it can rely on without a +// confirming grep. Empty when there is nothing listed. +func trustLine(ids []int, score map[int]float64) string { + if len(ids) == 0 { + return "" + } + sure := 0 + for _, v := range ids { + if score[v] >= 0.9 { + sure++ + } + } + if sure == len(ids) { + return fmt.Sprintf("trust: all %d sure, no confirming grep needed", sure) + } + return fmt.Sprintf("trust: %d of %d sure, grep to confirm the %d likely", sure, len(ids), len(ids)-sure) +} diff --git a/internal/graph/view_test.go b/internal/graph/view_test.go index 34236db..1075e2f 100644 --- a/internal/graph/view_test.go +++ b/internal/graph/view_test.go @@ -96,7 +96,7 @@ func TestPathAndImpact(t *testing.T) { t.Fatalf("unexpected path between disconnected symbols: %+v", p) } im, _ := g.Impact("check", 4, 2000, false) - for _, want := range []string{"validate", "ParseConfig"} { + for _, want := range []string{"validate", "ParseConfig", "trust: "} { if !strings.Contains(im, want) { t.Errorf("impact missing %s:\n%s", want, im) } diff --git a/internal/setup/digest.go b/internal/setup/digest.go index a426d48..3509893 100644 --- a/internal/setup/digest.go +++ b/internal/setup/digest.go @@ -79,6 +79,15 @@ func IsSmallRepo(root string, idx *schema.Index) bool { return limit > 0 && n > 0 && n < limit } +// IsQuietRepo reports whether hooks should inject nothing for this repo. The +// small repo limit still applies on top, so a quiet limit of 0 leaves small +// repos with their one line. +func IsQuietRepo(root string, idx *schema.Index) bool { + limit := config.Load(root).QuietRepoLimit() + n := SourceFiles(idx) + return limit > 0 && n > 0 && n < limit +} + // smallRepoLine is the only orientation a small repo gets. It is factual, not // imperative, like the other hook text. func smallRepoLine(idx *schema.Index) string { @@ -89,7 +98,7 @@ func smallRepoLine(idx *schema.Index) string { // (the note plus the structure token, so the session hook still recognises it), // every other repo gets the full digest. func BodyForRoot(root string, idx *schema.Index, variant string) string { - if !IsSmallRepo(root, idx) { + if !IsSmallRepo(root, idx) && !IsQuietRepo(root, idx) { return BodyFor(idx, variant) } return fmt.Sprintf("## Codebase map (stacklit)\n%s\n\n```\n%s | %s | %d modules | %s%s\n```\n", diff --git a/internal/setup/session.go b/internal/setup/session.go index 6b1fe84..454f43b 100644 --- a/internal/setup/session.go +++ b/internal/setup/session.go @@ -119,6 +119,9 @@ func SessionHook(stdin io.Reader) string { hash := session.StructureHash(idx) var parts []string switch { + case IsQuietRepo(root, idx): + session.RecordInjected(dir, in.SessionID, hash) + return "" case IsSmallRepo(root, idx): parts = append(parts, smallRepoLine(idx)) case rulesHash(root) != hash: @@ -177,6 +180,9 @@ func SubagentHook(stdin io.Reader) string { if err != nil { return "" } + if IsQuietRepo(root, idx) { + return "" + } if IsSmallRepo(root, idx) { return hookJSON("SubagentStart", smallRepoLine(idx)) } diff --git a/internal/setup/small_test.go b/internal/setup/small_test.go index 5736bba..dee05e8 100644 --- a/internal/setup/small_test.go +++ b/internal/setup/small_test.go @@ -46,6 +46,7 @@ func TestIsSmallRepoThresholdAndSwitches(t *testing.T) { } func TestSessionHookSmallRepoGetsOneLineNotTheDigest(t *testing.T) { + t.Setenv("STACKLIT_QUIET_REPO_FILES", "0") t.Setenv("STACKLIT_CACHE_DIR", t.TempDir()) root := t.TempDir() idx := idxWithFiles(100) @@ -81,6 +82,7 @@ func TestSessionHookSmallRepoGetsOneLineNotTheDigest(t *testing.T) { } func TestSubagentHookSmallRepo(t *testing.T) { + t.Setenv("STACKLIT_QUIET_REPO_FILES", "0") t.Setenv("STACKLIT_CACHE_DIR", t.TempDir()) root := t.TempDir() writeIdx(t, root, idxWithFiles(100)) @@ -113,3 +115,25 @@ func TestBodyForRootSmallBlockIsShortAndRecognisable(t *testing.T) { } t.Logf("small block %d tokens, full digest %d tokens", Tokens(sb), Tokens(BodyFor(small, "e"))) } + +func TestQuietRepoHooksInjectNothing(t *testing.T) { + t.Setenv("STACKLIT_CACHE_DIR", t.TempDir()) + root := t.TempDir() + writeIdx(t, root, idxWithFiles(3000)) + if !IsQuietRepo(root, idxWithFiles(3000)) || IsQuietRepo(root, idxWithFiles(5000)) { + t.Fatal("default quiet limit is exclusive at 5000") + } + for _, src := range []string{"startup", "resume", "compact"} { + if out := SessionHook(payload(t, root, src, "q-"+src)); out != "" { + t.Fatalf("%s: quiet repo must inject nothing: %q", src, out) + } + } + in := strings.NewReader(`{"cwd":"` + root + `","agent_type":"Explore"}`) + if out := SubagentHook(in); out != "" { + t.Fatalf("quiet repo subagent must get nothing: %q", out) + } + t.Setenv("STACKLIT_QUIET_REPO_FILES", "0") + if out := SessionHook(payload(t, root, "startup", "q-off")); out == "" { + t.Fatal("quiet mode off must restore the hook text") + } +} From ae9e5722ed3d0fcaf7c06f4a62a748dfbee85723 Mon Sep 17 00:00:00 2001 From: GDS K S Date: Sun, 8 Nov 2026 20:50:28 -0600 Subject: [PATCH 2/5] docs: document quiet repo mode and the impact trust line --- CHANGELOG.md | 2 ++ docs/USAGE.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1ea438..d96c066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Follows the first real-agent A/B (bench/agent-ab, pull request #77), which found - `query` adds one line naming the next ranked files and the `explain` and `path` commands when the question words suggest a flow or chain, or the top files are scattered with flat scores. It stays inside the token budget and leaves single-answer queries unchanged. `STACKLIT_QUERY_NEXT` (0 turns it off), `STACKLIT_FLOW_WORDS`, `STACKLIT_QUERY_SPREAD_DIRS`, `STACKLIT_QUERY_SPREAD_TOP`, `STACKLIT_QUERY_FLATNESS`. - `impact` on a file or directory lists its importers as `path:line`, with the sources checked for script languages so the list matches a grep, and `--calls` for the previous call-level view. A Go package directory used to be an error. About 40 to 160 tokens for a package against 110 to 390 for `grep -rn` of the same import. `STACKLIT_IMPACT_SCAN`, `STACKLIT_IMPACT_SCAN_MAX`. - Small repo mode: below `small_repo_files` (default 600, `STACKLIT_SMALL_REPO_FILES`, 0 off) source files, the session hook, subagent hook and `setup` carry a one line note that grep is enough instead of the digest (about 60 tokens instead of 490). +- Quiet repo mode: below `quiet_repo_files` (default 5000, `STACKLIT_QUIET_REPO_FILES`, 0 off) source files, the session and subagent hooks inject nothing at all. Round 2 measured Stacklit as costlier at 412 to 2,919 files and cheaper only at 15,978, so the 5000 default sits between and is not yet measured. The `setup` rules block stays the one line note. +- `impact` prints a `trust:` line saying how many listed dependents are sure (score 0.9 or more) and how many need a confirming grep. - A pre-registered second A/B round in `bench/agent-ab/tasks-round2/`, with a `stacklit-e` arm in the harness. It has not been run. ### Changed diff --git a/docs/USAGE.md b/docs/USAGE.md index 46d1a59..a5b531f 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -519,6 +519,7 @@ Create `.stacklitrc.json` in the project root. The file is optional, and every k | `timestamps` | false | Writes `generated_at`. Off by default, so a committed index changes only when the code does. Env: `STACKLIT_TIMESTAMPS=1`. | | `git_activity` | false | Adds git hot and recent files to the index. Off by default for the same reason. Turn it on with this key or with `STACKLIT_GIT=1`. | | `steering` | `a` | Wording of the rules at the top of the generated `CLAUDE.md` or `AGENTS.md` block: `a`, `b`, `c`, `d` (strict, measured worse, not recommended) or `e` (routed, not yet measured). See [docs/SESSIONS.md](docs/SESSIONS.md). Env: `STACKLIT_STEERING`. | +| `quiet_repo_files` | 5000 | A repo with fewer indexed source files than this gets no hook text at all from the session and subagent hooks. 0 turns it off. Env: `STACKLIT_QUIET_REPO_FILES`. | | `small_repo_files` | 600 | A repo with fewer indexed source files than this gets a one-line note instead of the digest, in the session hook and in `setup`. 0 turns it off. Env: `STACKLIT_SMALL_REPO_FILES`. | | `crawl_guard` | (installed mode) | Crawl guard mode: `off`, `nudge` or `redirect`. Env: `STACKLIT_CRAWL_GUARD`. | | `stats` | false | Appends a local usage log of `query`, `slice`, `explain` and `impact` calls. Env: `STACKLIT_STATS=1`. Never sent over the network. | @@ -555,6 +556,7 @@ Only files with recognised source extensions are scanned. Directories named `tes | `STACKLIT_ADAPTER_TIMEOUT_SEC`, `STACKLIT_ADAPTER_MAX_BYTES`, `STACKLIT_ADAPTER_MAX_ITEMS` | `ingest --adapter` | Adapter timeout, output size and item caps. Defaults: 60, 4194304, 5000. | | `STACKLIT_DIGEST_TOKENS` | `setup`, hooks | Token cap of the injected digest. Default 500. | | `STACKLIT_STEERING` | `setup`, `guide`, `serve` | Steering wording variant `a`, `b`, `c`, `d` or `e`. `d` also selects the older MCP and guide wording. | +| `STACKLIT_QUIET_REPO_FILES` | hooks | Source file count below which the session and subagent hooks stay silent. Default 5000, 0 off. | | `STACKLIT_SMALL_REPO_FILES` | `setup`, hooks | Source file count below which a repo gets the small repo note instead of the digest. Default 600, 0 off. | | `STACKLIT_QUERY_NEXT`, `STACKLIT_FLOW_WORDS`, `STACKLIT_QUERY_SPREAD_DIRS`, `STACKLIT_QUERY_SPREAD_TOP`, `STACKLIT_QUERY_FLATNESS` | `query` | The "also ranked" line: files listed (default 4, 0 off), flow word list, and the spread test (defaults 4, 5, 0.5). | | `STACKLIT_IMPACT_SCAN`, `STACKLIT_IMPACT_SCAN_MAX` | `impact` | `0` stops `impact` from reading script sources to confirm importers. The cap sets how many files are read (default 20000). | From af41b8a12f4844caeee9bd2b331aa84a8340c695 Mon Sep 17 00:00:00 2001 From: GDS K S Date: Sun, 8 Nov 2026 21:20:45 -0600 Subject: [PATCH 3/5] docs: add Graphify to the comparison without making measured claims --- docs/COMPARISON.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index 71c6e01..89036eb 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -2,7 +2,7 @@ This page compares the main ways to give an AI agent codebase context. -Last updated: 2026-04-10. [Submit corrections](https://github.com/glincker/stacklit/issues). +Last updated: 2026-04-10 (Graphify added 2026-11-08, other rows not rechecked). [Submit corrections](https://github.com/glincker/stacklit/issues). ## The problem @@ -21,6 +21,7 @@ Tools solve this in different ways. Some dump everything. Some build knowledge g | [Aider repo-map](https://github.com/Aider-AI/aider) | 43k** | Tree-sitter + PageRank | Text | ~1k | 40+ | Yes | Locked to Aider | | [Codebase-Memory](https://github.com/DeusData/codebase-memory-mcp) | 1.4k | Knowledge graph | SQLite | 2k-10k/session | 66 | Yes (call graph) | MCP server (C) | | [Axon](https://github.com/harshkedia177/axon) | 648 | Graph + community detection | Neo4j/KuzuDB | 2k-10k/session | 3 | Yes (blast radius) | MCP server (Python) | +| [Graphify](https://github.com/Graphify-Labs/graphify) | 144k | Tree-sitter knowledge graph, plus docs and media | graph.json, HTML, report | not measured here | ~40 (code) | Yes (calls, imports, inherits) | Skill and CLI (Python), paid cloud tier | | **[Stacklit](https://github.com/glincker/stacklit)** | -- | Tree-sitter + module map | JSON + HTML | **137 to 1,351 (compact map on six repos; scales with module count)** | 48 | Yes | CLI (Go) | *Token cost = tokens consumed to give an agent full project context on a ~10k-line repo. @@ -67,6 +68,15 @@ Stacklit parses code with tree-sitter, builds a module-level dependency graph, a Best for: any repo, any AI tool, and no ongoing maintenance. +### Graphify + +Closest in design to Stacklit's symbol graph. Its README describes local tree-sitter parsing with no LLM for code, edges tagged `EXTRACTED` or `INFERRED`, `query`, `path` and `explain` commands, and communities. Stacklit has the same edge tags and the same three commands. Differences, per its README and not measured by us: + +- It also maps docs, PDFs, images and video, using a model for that part. Stacklit indexes code only. +- Its install is a Python tool plus an assistant skill. Stacklit is one Go binary with a committable `stacklit.json`. +- Its 80 percent token claim is for its paid cloud product. Its published code benchmark is 6 questions on one repo (82.0 percent key-fact coverage against 70.8 for grep and read), too small to compare with ours. +- We have not run it on our benchmark. No parity number exists yet, so this page makes no speed, size or accuracy claim against it. + ### IDE-integrated (proprietary) **Cursor, Continue.dev, GitHub Copilot, Sourcegraph Cody** From 9e3484a7c77bae1fee77f3208a54c5bb5a894439 Mon Sep 17 00:00:00 2001 From: GDS K S Date: Sun, 8 Nov 2026 21:37:11 -0600 Subject: [PATCH 4/5] feat: trust advice on path output --- CHANGELOG.md | 1 + internal/graph/path.go | 10 +++++++++- internal/graph/view_test.go | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d96c066..b2b0e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Follows the first real-agent A/B (bench/agent-ab, pull request #77), which found - Small repo mode: below `small_repo_files` (default 600, `STACKLIT_SMALL_REPO_FILES`, 0 off) source files, the session hook, subagent hook and `setup` carry a one line note that grep is enough instead of the digest (about 60 tokens instead of 490). - Quiet repo mode: below `quiet_repo_files` (default 5000, `STACKLIT_QUIET_REPO_FILES`, 0 off) source files, the session and subagent hooks inject nothing at all. Round 2 measured Stacklit as costlier at 412 to 2,919 files and cheaper only at 15,978, so the 5000 default sits between and is not yet measured. The `setup` rules block stays the one line note. - `impact` prints a `trust:` line saying how many listed dependents are sure (score 0.9 or more) and how many need a confirming grep. +- `path` adds advice after the weakest hop score: sure (0.9 or more) needs no confirming grep, otherwise grep the weak hop. - A pre-registered second A/B round in `bench/agent-ab/tasks-round2/`, with a `stacklit-e` arm in the harness. It has not been run. ### Changed diff --git a/internal/graph/path.go b/internal/graph/path.go index 64a43e2..7fdb115 100644 --- a/internal/graph/path.go +++ b/internal/graph/path.go @@ -127,7 +127,7 @@ func (p *PathResult) Render(asJSON bool) string { fmt.Fprintf(&b, "path %s -> %s: %s\n", p.From, p.To, p.Note) return b.String() } - fmt.Fprintf(&b, "path %s -> %s: %d hops, weakest hop %.2f\n", p.From, p.To, len(p.Hops), p.Weakest) + fmt.Fprintf(&b, "path %s -> %s: %d hops, weakest hop %.2f (%s)\n", p.From, p.To, len(p.Hops), p.Weakest, pathTrust(p.Weakest)) if p.Note != "" { b.WriteString(p.Note + "\n") } @@ -144,3 +144,11 @@ func (p *PathResult) Render(asJSON bool) string { } return b.String() } + +// pathTrust turns the weakest hop score into advice on whether to confirm. +func pathTrust(weakest float64) string { + if weakest >= 0.9 { + return "sure, no confirming grep needed" + } + return "likely, grep to confirm the weak hop" +} diff --git a/internal/graph/view_test.go b/internal/graph/view_test.go index 1075e2f..46ac8e9 100644 --- a/internal/graph/view_test.go +++ b/internal/graph/view_test.go @@ -91,6 +91,9 @@ func TestPathAndImpact(t *testing.T) { if err != nil || !p.Found || len(p.Hops) != 2 || p.Weakest <= 0 { t.Fatalf("path = %+v err=%v", p, err) } + if r := p.Render(false); !strings.Contains(r, "weakest hop") || !strings.Contains(r, "grep") { + t.Fatalf("path render lacks trust advice: %s", r) + } p, _ = g.ShortestPath("ParseConfig", "Serve") if p.Found { t.Fatalf("unexpected path between disconnected symbols: %+v", p) From 4721e6a9a7627ba57d9f02e58e3b4b71fa17eb1c Mon Sep 17 00:00:00 2001 From: GDS K S Date: Sun, 8 Nov 2026 22:03:08 -0600 Subject: [PATCH 5/5] test: pre-register round 3 of the agent test and add the quiet arm --- bench/agent-ab/PREREGISTRATION-round3.md | 62 ++++++++++++++++++++++++ bench/agent-ab/main.go | 4 +- bench/agent-ab/main_test.go | 9 ++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 bench/agent-ab/PREREGISTRATION-round3.md diff --git a/bench/agent-ab/PREREGISTRATION-round3.md b/bench/agent-ab/PREREGISTRATION-round3.md new file mode 100644 index 0000000..9a156a5 --- /dev/null +++ b/bench/agent-ab/PREREGISTRATION-round3.md @@ -0,0 +1,62 @@ +# Pre-registration, round 3: does quiet repo mode remove the cost on repos below 5,000 files? + +Written and committed before any round 3 session ran. Spend approval is recorded under "Approval". Branch: `feat/gate-and-trust`. + +## Why + +Round 2 (`RESULTS-round2.md`) found Stacklit cost more than no Stacklit on the three repos below 5,000 files: stacklit-e against control was 1.05 on theauth-go, 1.21 on featuredrop and 1.42 on theauth. It only paid off on levelrail (15,978 files, 0.87). Quiet repo mode (default `quiet_repo_files` 5000) makes the session and subagent hooks inject nothing below that size. This round asks one question: with quiet mode on, does a session on those repos cost the same as one without Stacklit? + +## What this round cannot answer + +Where the break-even sits between 2,919 and 15,978 files. No repo we own or can use here has between 3,000 and 15,000 source files (see the repo notes in `tasks-round2/PREREGISTRATION-round2.md`), so the 5000 default stays an unmeasured guess above 2,919 files. Famous public repos would fill the gap but need tasks on code newer than the model's knowledge (issue #83) and a separate approval. + +## Arms + +| Arm | Setup | Environment | +|---|---|---| +| `control` | nothing added; tools Read, Grep, Glob | none | +| `stacklit-quiet` | `stacklit generate --graph`, then `stacklit setup claude --session-hook --no-mcp --no-git-hook`; Bash only as `Bash(stacklit *)` | `STACKLIT_STEERING=e`, `STACKLIT_QUIET_REPO_FILES=5000`, `STACKLIT_SMALL_REPO_FILES=600` | + +The binary is built from the head of `feat/gate-and-trust`; its SHA-256 prefix is in every session record. Session settings are those of round 2. + +## Repos and tasks + +theauth-go (1,294 files), featuredrop (412) and theauth (2,919), at the round 2 SHAs in `repos-round2.json`. levelrail is left out because it is above the quiet limit, so the arm would equal `stacklit-e`, already measured. The tasks are the round 2 tasks (`tasks-round2/`, 10 per repo, 30 in all). They are not new to the model's harness but were not used to tune quiet mode, which does not touch task content. Round 2 found featuredrop contaminated (3 of 6 right with tools off), so every result is reported on all three repos and without featuredrop. + +## Size, cost and caps + +2 arms x 30 tasks x 5 trials = 300 sessions. Projection at 0.025 USD per session (round 2 mean 0.022 to 0.026): 7.50 USD notional. Caps: 0.25 USD per session, 25 turns, `--cap-usd 15` for the ledger. Stop at once on a rate limit, usage limit or authentication error; stop after 10 consecutive harness failures. No interim analysis and no stopping for efficacy. If the run stops early, the decision rule is not evaluated. Order is shuffled with seed 20261109. Bootstrap seed 20260508, 10,000 resamples, paired by task and trial. No new memorisation check: the round 2 check covers these repos and tasks. + +## Grading + +The round 2 matcher, unchanged. 10 percent of transcripts are checked by hand. A defect found afterwards is reported as a separate column and is not used by the decision rule. + +## Decision rule (fixed before seeing data) + +Quiet mode counts as "no regression below 5,000 files" if and only if all hold, pooled over the three repos (150 pairs): + +| Id | Criterion | +|---|---| +| Q1 | Cost ratio `stacklit-quiet` / `control`, upper 95% bound below 1.05 | +| Q2 | Correctness difference (`stacklit-quiet` minus `control`), lower 95% bound above -5 points | +| Q3 | On each repo, cost ratio upper 95% bound below 1.15 | + +If any fails, the default quiet limit is wrong for that size and the result says which repo. If all pass, the supported claim is only: "on three repos of 412 to 2,919 files, sessions with quiet-mode Stacklit cost no more than sessions without it". It is not a savings claim, and it says nothing about repos between 3,000 and 15,000 files. + +## Limits + +One model, one machine, tasks written by us on repos we own, path-match correctness, notional cost, small n per cell, round 2 tasks reused. The setup still writes a one line note into CLAUDE.md in quiet repos, so this tests quiet mode as shipped and not a fully silent install. + +## Run + +```bash +go build -o /tmp/stacklit ./cmd/stacklit && go build -o /tmp/agent-ab ./bench/agent-ab +R3="-repos bench/agent-ab/repos-round2.json -tasks bench/agent-ab/tasks-round2 -results bench/agent-ab/results-round3 -ledger bench/agent-ab/round3/ledger.jsonl -stacklit /tmp/stacklit -model claude-sonnet-5-5 -seed 20261109 -max-budget-usd 0.25 -cap-usd 15" +/tmp/agent-ab -repo theauth-go,featuredrop,theauth $R3 -arms control,stacklit-quiet -trials 5 -dry-run +/tmp/agent-ab -repo theauth-go,featuredrop,theauth $R3 -arms control,stacklit-quiet -trials 5 -yes-spend +/tmp/agent-ab -analyze $R3 +``` + +## Approval + +The owner replied "ok" on 2026-11-08 to a proposed cap of 15 USD notional for a re-measure of the gate. This round spends at most that. The round 3 run starts only after this file and the harness change are committed. diff --git a/bench/agent-ab/main.go b/bench/agent-ab/main.go index 84d2640..ccbb6cd 100644 --- a/bench/agent-ab/main.go +++ b/bench/agent-ab/main.go @@ -115,7 +115,7 @@ func main() { flag.StringVar(&c.model, "model", "sonnet", "model alias passed to claude") flag.StringVar(&c.resultsDir, "results", "bench/agent-ab/results", "directory for per-session JSON") flag.StringVar(&c.ledger, "ledger", "bench/agent-ab/ledger.jsonl", "append-only cost ledger") - flag.StringVar(&c.arms, "arms", "control,stacklit", "arms to run: control, stacklit, strict, stacklit-e, stacklit-e-small") + flag.StringVar(&c.arms, "arms", "control,stacklit", "arms to run: control, stacklit, strict, stacklit-e, stacklit-e-small, stacklit-quiet") flag.StringVar(&c.grader, "grader", "audited", "answer matcher: audited (accepts ./path and absolute checkout paths, round 2) or prereg (round 1)") flag.StringVar(&c.smallRepos, "small-repos", "featuredrop", "repos where the stacklit-e-small arm runs (the others are above the small repo limit)") flag.IntVar(&c.smallFiles, "small-repo-files", 600, "STACKLIT_SMALL_REPO_FILES value for the stacklit-e-small arm") @@ -468,6 +468,8 @@ func armEnv(c config, arm string) []string { return []string{"STACKLIT_STEERING=a", "STACKLIT_SMALL_REPO_FILES=0"} case "stacklit-e": return []string{"STACKLIT_STEERING=e", "STACKLIT_SMALL_REPO_FILES=0"} + case "stacklit-quiet": + return []string{"STACKLIT_STEERING=e", "STACKLIT_QUIET_REPO_FILES=5000", "STACKLIT_SMALL_REPO_FILES=600"} case "stacklit-e-small": return []string{"STACKLIT_STEERING=e", fmt.Sprintf("STACKLIT_SMALL_REPO_FILES=%d", c.smallFiles)} } diff --git a/bench/agent-ab/main_test.go b/bench/agent-ab/main_test.go index 757cc5f..56732f2 100644 --- a/bench/agent-ab/main_test.go +++ b/bench/agent-ab/main_test.go @@ -328,3 +328,12 @@ func TestDecisionReportRule(t *testing.T) { t.Error("no data must keep the default") } } + +func TestArmEnvQuietKeepsTheDefaultLimits(t *testing.T) { + env := strings.Join(armEnv(config{}, "stacklit-quiet"), " ") + for _, want := range []string{"STACKLIT_QUIET_REPO_FILES=5000", "STACKLIT_SMALL_REPO_FILES=600", "STACKLIT_STEERING=e"} { + if !strings.Contains(env, want) { + t.Fatalf("quiet arm env %q lacks %s", env, want) + } + } +}