From 493a48041a38b99429598c11e088955b8c12f080 Mon Sep 17 00:00:00 2001 From: argszero Date: Tue, 20 Oct 2026 08:45:08 +0800 Subject: [PATCH 1/2] emrg: the install drift reading says which release the tree is The tool asked whether a file's content is reachable from any ref a release is built from, which a five-release-old install tree passes: the old bytes are in an older release's history, and that is a release ref. So a tree that renders a superseded evolution prompt read as shipped. Ask the other question too - which release this tree is and which is newest - and print it under both verdicts, so a green drift verdict cannot hide it. --- scripts/check-install-drift.py | 198 ++++++++++++++++++++++++++++-- tests/test_check_install_drift.py | 157 +++++++++++++++++++++++ 2 files changed, 343 insertions(+), 12 deletions(-) diff --git a/scripts/check-install-drift.py b/scripts/check-install-drift.py index b2d77349..1849f207 100644 --- a/scripts/check-install-drift.py +++ b/scripts/check-install-drift.py @@ -76,6 +76,18 @@ only in such a ref is therefore reported - and the report **names the ref**, because that is where the edit is still recoverable from. +**Which release that is gets its own reading, because the one above cannot see it.** Everything +above is true of a tree five releases old - its files match the commits that shipped them - so +the green verdict is silent about the daemon rendering that release's rules rather than the +newest ones. Measured 2026-10-20: this host's install tree carries +`emrg/server/evolution_prompt.md`'s pre-narrowing blob (the one `v0.3.14`-`v0.3.16` ship), five +releases past the host's 2026-10-16 ruling that narrowed the abstention clause, and the reading +above answered `0`. Every evolution cycle that day read the superseded rule out of the prompt +the daemon rendered, and none of them could see it: nothing measured it. So a `behind:` line is +printed under **both** verdicts - which release this tree is, which release is newest, and the +paths that differ. Being behind is not drift (nothing is lost) and not clean (it is what runs), +so it is stated rather than scored, and the exit code stays the drift reading's alone. + The walk itself is asked with `--missing=print` and read for the objects it does list. A clone can hold a ref whose history it cannot fully read: measured on this host 2026-10-08 (`cyc20261008-001036`), where `git rev-list --all --objects` exits 128 on `bad tree object @@ -128,6 +140,7 @@ from __future__ import annotations import argparse +import re import subprocess import sys import traceback @@ -192,6 +205,101 @@ def tracked_files(root: Path) -> set[str]: return {name for name in _git(root, "ls-files", "-z").split("\0") if name} +#: The spelling the release workflow gives a published version, and the only refs the +#: "which release is this install tree" question may be answered from. +RELEASE_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$") + + +def release_tags(root: Path) -> list[str]: + """Every release tag this clone carries, newest first. + + Version order, not creation order: `--sort=-v:refname` is git's own version sort, which + puts `v0.3.10` above `v0.3.9` where a lexical sort would put it below - the wrong sort does + not crash, it names a real release that is simply not the newest. A tag this clone does not + have cannot be named, and that is reported rather than guessed (see `behind_release`). + """ + listing = _git(root, "tag", "--sort=-v:refname", "--list", "v*") + return [name for name in (line.strip() for line in listing.splitlines()) if RELEASE_TAG_RE.match(name)] + + +def release_tree(root: Path, ref: str) -> dict[str, str]: + """`{path: blob id}` for one release ref, from a single `ls-tree`. + + One call for the whole tree rather than a `rev-parse` per path: the question compares + every shared file at once, and this host's install tree shares ~380 of them, which at one + process each is a second of subprocesses per release examined. + """ + blobs: dict[str, str] = {} + for line in _git(root, "ls-tree", "-r", ref).splitlines(): + if "\t" not in line: + continue + meta, path = line.split("\t", 1) + parts = meta.split() + if len(parts) >= 3 and parts[1] == "blob": + blobs[path] = parts[2] + return blobs + + +def behind_release( + install_dir: Path, root: Path, shared: list[tuple[Path, str]] +) -> tuple[str | None, str | None, list[str], int]: + """`(newest tag, the tag this tree is, paths that differ from the newest, shared count)`. + + The question the rest of this tool cannot answer, and the reason it is here: `scan` asks + whether a file's content is reachable from *any* ref a release is built from. An install + tree several releases old passes that test with a green verdict, because the old bytes are + in an older release's history - which is a release ref. Measured 2026-10-20 on this host: + the install tree carries `evolution_prompt.md`'s pre-narrowing blob (`af6e4ae5`, the one + `v0.3.14`-`v0.3.16` ship) while master and the five releases since carry the narrowed rule, + and the other reading answered `0` - "every file ... holds content the refs a release is + built from still reach". The daemon renders that file into every evolution cycle, so the + reading that said "shipped" said it about a superseded rule. + + So the second question is: **which release is this tree, and is a newer one out?** It is + answered for the tree as a whole rather than file by file, and that distinction is + measured: attributing each file to "the newest release that still carries these bytes" + names a *later* tag for every file the next release did not touch, so a v0.3.16 tree + reported `v0.3.16` for one path and `v0.3.18` for another and named no version at all. The + tree's version is the newest tag whose content it matches on every shared path, which is + one fact about one directory. + + A tree matching no tag is `None` and is not this reading's fault to explain: content in no + tag is the *other* reading's subject (a hand edit, which destroys on upgrade), and mixing + the two in one line leaves a reader unable to tell "an upgrade brings the rest" from "an + upgrade destroys this". + + Absent files are out of scope: this compares the paths both trees carry, so a file the + newest release added is something an upgrade creates, not a lag in what this tree has. + + `None` for the newest tag is "this clone carries no release tag", which the caller must + report as unmeasurable rather than as currency - `[]` with a tag is "measured, nothing lags". + """ + if not shared: + return None, None, [], 0 + tags = release_tags(root) + if not tags: + return None, None, [], len(shared) + + ids = {relative: _git_stored_id(root, path, relative) for path, relative in shared} + # The tree's own version: the newest release it matches on every shared path. Walked from + # the newest so the first match is the answer, and the walk stops there. + install_release: str | None = None + for tag in tags: + blobs = release_tree(root, tag) + if all(blobs.get(relative) == blob for relative, blob in ids.items()): + install_release = tag + break + + newest = tags[0] + newest_blobs = release_tree(root, newest) + differing = sorted( + relative + for relative, blob in ids.items() + if newest_blobs.get(relative) != blob + ) + return newest, install_release, differing, len(shared) + + #: The ref namespaces a release is built from. Everything else a ref can live in #: (`refs/emrg/rescue/*`, `refs/stash`, `refs/emrg-tmp/*`, `refs/emrg-check/*`, ...) is this #: machinery's own bookkeeping, and a recovery snapshot above all: `scripts/recover-worktree.py` @@ -267,20 +375,13 @@ def refs_holding(root: Path, refs: list[str], blobs: set[str]) -> dict[str, list return found -def scan( - install_dir: Path, root: Path -) -> tuple[list[tuple[str, int, str]], list[str], int, int]: - """Return `(drifted, skipped, checked, missing)` for the files the two trees share. +def shared_paths(install_dir: Path, root: Path) -> tuple[list[tuple[Path, str]], list[str]]: + """`(install path, install-relative path)` for the paths both trees carry, and the rest. - `drifted` is `(path, size, note)` for a shared file whose content no shipped object - holds - the note saying why, and naming the bookkeeping ref it is still recoverable - from when one holds it; `skipped` is the install-relative paths this checkout no longer - tracks; `checked` is how many files the question was actually asked about, which `main` - refuses to report a green verdict over when it is 0; `missing` is how many objects the - membership walk found absent from this clone, which `main` prints when it is not zero. + The walk both readings start from: a path is comparable only when this checkout tracks it, + because the id of the other side is asked of git at *that* path. """ tracked = tracked_files(root) - shipped, missing = shipping_objects(root) shared: list[tuple[Path, str]] = [] skipped: list[str] = [] for path in sorted(install_dir.rglob("*")): @@ -291,6 +392,74 @@ def scan( skipped.append(relative) continue shared.append((path, relative)) + return shared, skipped + + +#: How many lagging paths a `behind:` line names before counting the rest, matching the +#: leftovers section's shape: enough to recognise the family, not a wall of paths. +BEHIND_SHOWN = 10 + + +def _report_release( + install_dir: Path, + root: Path, + newest: str | None, + install_release: str | None, + differing: list[str], + checked: int, +) -> None: + """Print which release this tree is, and whether a newer one exists. + + Printed under **both** verdicts, because it is a second question and not a verdict: a tree + that lags is not drift (nothing here is lost) and is not clean either (the daemon renders + it). Leaving it out of the green branch is the defect this function exists to fix - the + green line said "a release's refs still reach this content", which is true of any older + release and therefore said nothing about being current. + """ + if newest is None: + print( + f"behind: could not measure which release {install_dir} is - this checkout " + "carries no release tag, and the refs a release is built from cannot answer it " + "(an older release's bytes are in them too)" + ) + return + if not differing: + print( + f"behind: none - {install_dir} carries what {newest} ships " + f"({checked} shared file(s) compared)" + ) + return + + version = install_release or "no single release" + print( + f"behind: {install_dir} is the {version} copy and the newest release is {newest}; " + f"{len(differing)} of {checked} shared file(s) differ, so whatever the daemon renders " + "from this tree is that release's copy" + ) + for relative in differing[:BEHIND_SHOWN]: + print(f" {relative}") + if len(differing) > BEHIND_SHOWN: + print(f" ... and {len(differing) - BEHIND_SHOWN} more") + print( + f" remedy: an upgrade to {newest} brings these. Until then a rule read from this " + "tree is the installed release's rule, not the newest one's." + ) + + +def scan( + install_dir: Path, root: Path +) -> tuple[list[tuple[str, int, str]], list[str], int, int]: + """Return `(drifted, skipped, checked, missing)` for the files the two trees share. + + `drifted` is `(path, size, note)` for a shared file whose content no shipped object + holds - the note saying why, and naming the bookkeeping ref it is still recoverable + from when one holds it; `skipped` is the install-relative paths this checkout no longer + tracks; `checked` is how many files the question was actually asked about, which `main` + refuses to report a green verdict over when it is 0; `missing` is how many objects the + membership walk found absent from this clone, which `main` prints when it is not zero. + """ + shared, skipped = shared_paths(install_dir, root) + shipped, missing = shipping_objects(root) drifted: list[tuple[str, int, str]] = [] for path, relative in shared: @@ -356,7 +525,8 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( "Files in the live install tree whose content the refs a release is built from " - "do not hold: content an upgrade replaces." + "do not hold (content an upgrade replaces), and which release that tree is " + "(content an upgrade brings)." ) ) parser.add_argument( @@ -399,6 +569,8 @@ def main(argv: list[str] | None = None) -> int: return 2 try: drifted, skipped, checked, missing = scan(install_dir, root) + shared, _ = shared_paths(install_dir, root) + newest, install_release, differing, _ = behind_release(install_dir, root, shared) except RuntimeError as exc: print(f"could not measure: {exc}", file=sys.stderr) return 2 @@ -466,6 +638,7 @@ def main(argv: list[str] | None = None) -> int: f"{len(leftover)} named as leftovers above, " f"{len(caches)} interpreter cache file(s) not listed)" ) + _report_release(install_dir, root, newest, install_release, differing, checked) return 0 @@ -477,6 +650,7 @@ def main(argv: list[str] | None = None) -> int: "install/upgrade replaces it whole, so the next one destroys it: move the change " "into the source checkout and ship it, or it is lost." ) + _report_release(install_dir, root, newest, install_release, differing, checked) return 1 diff --git a/tests/test_check_install_drift.py b/tests/test_check_install_drift.py index f18d494e..e166c058 100644 --- a/tests/test_check_install_drift.py +++ b/tests/test_check_install_drift.py @@ -719,3 +719,160 @@ def test_interpreter_caches_are_counted_but_not_listed( ) assert "1 interpreter cache file(s) not listed" in done.stdout, done.stdout assert "chat.pyc" not in done.stdout, done.stdout + + +# ── the second question: which release is this tree, and is a newer one out? ───────────── +# +# `scan` asks whether a file's content is reachable from any ref a release is built from, and +# an install tree several releases old passes it: the old bytes are in an older release's +# history, which is a release ref. Measured 2026-10-20: this host's install tree carries the +# pre-narrowing `evolution_prompt.md` while master and five releases since carry the narrowed +# rule, and the drift reading answered `0`. The cases below pin the reading that can see it. + + +def _recommit(root: Path, text: str, message: str) -> None: + """Change `SHARED` and commit, so each commit can carry its own release tag.""" + (root / SHARED).write_text(text, encoding="utf-8") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", message) + + +def test_a_tree_that_is_an_older_release_is_named_as_behind( + install_dir: Path, checkout: Path +) -> None: + """The measured defect: a green drift verdict over a superseded install tree. + + The tree holds `v0.3.1`'s bytes, `v0.3.2` changed them, and the drift reading still says + every file is reachable from a release ref - which is true, and is exactly why it cannot + answer this question. The `behind:` line has to name both versions, because "behind" with + no number is a warning nobody can act on. + """ + _git(checkout, "tag", "v0.3.1") + _recommit(checkout, "narrowed content\n", "narrow the rule") + _git(checkout, "tag", "v0.3.2") + + done = _run(install_dir, checkout) + + assert done.returncode == 0, done.stdout + done.stderr + assert "still reach" in done.stdout, "the drift reading is unchanged: " + done.stdout + assert "behind:" in done.stdout, done.stdout + assert "v0.3.1 copy" in done.stdout and "newest release is v0.3.2" in done.stdout, ( + "the line must name the tree's own release and the newest one: " + done.stdout + ) + assert SHARED in done.stdout, ( + "the lagging path must be named, or the reader cannot tell which rule is stale: " + + done.stdout + ) + assert "1 of 2 shared file(s) differ" in done.stdout, ( + "the size of the gap is a derived number, so it is asserted rather than left to a " + "reader to infer from a path list: " + done.stdout + ) + + +def test_a_tree_that_is_the_newest_release_is_not_reported_as_behind( + install_dir: Path, checkout: Path +) -> None: + """The other direction: a current tree must read as current, not merely stay silent. + + Silence and "current" are different readings and a reader has to be able to tell them + apart - which is why the up-to-date branch prints `none` and the version it compared to. + """ + _git(checkout, "tag", "v0.3.1") + + done = _run(install_dir, checkout) + + assert done.returncode == 0, done.stdout + done.stderr + assert "behind: none" in done.stdout, done.stdout + assert "carries what v0.3.1 ships" in done.stdout, done.stdout + assert "copy and the newest release" not in done.stdout, done.stdout + + +def test_a_clone_with_no_release_tag_says_it_could_not_measure( + install_dir: Path, checkout: Path +) -> None: + """No tags is `could not measure`, never "current". + + The rest of the tool answers a question about history and stays valid without tags, so the + exit code is unchanged; what must not happen is the second reading claiming a currency it + never measured. This is the family rule the whole tool is written to. + """ + done = _run(install_dir, checkout) + + assert done.returncode == 0, done.stdout + done.stderr + assert "could not measure which release" in done.stdout, done.stdout + assert "carries no release tag" in done.stdout, done.stdout + assert "behind: none" not in done.stdout, ( + "an unmeasured currency is not a clean one: " + done.stdout + ) + + +def test_the_newest_release_is_read_in_version_order_not_lexically( + install_dir: Path, checkout: Path +) -> None: + """`v0.3.10` is newer than `v0.3.9`; a lexical sort says the opposite. + + Pinned because the wrong sort does not crash or look odd - it names a real release that is + simply not the newest, and every subsequent sentence in the line inherits the mistake. + """ + _git(checkout, "tag", "v0.3.9") + _recommit(checkout, "content that ships in 0.3.10\n", "v0.3.10 content") + _git(checkout, "tag", "v0.3.10") + + done = _run(install_dir, checkout) + + assert done.returncode == 0, done.stdout + done.stderr + assert "newest release is v0.3.10" in done.stdout, done.stdout + assert "v0.3.9 copy" in done.stdout, done.stdout + + +def test_a_tree_matching_no_release_names_no_version(tmp_path: Path, checkout: Path) -> None: + """A mixed tree is not any release, and the line must not name one it does not match. + + Two files from two different releases: the tree is real and readable, so it is reported, + but calling it `v0.3.2` would be inventing a fact about which rule the daemon renders. + """ + _git(checkout, "tag", "v0.3.1") + _recommit(checkout, "shared content for 0.3.2\n", "second release") + _git(checkout, "tag", "v0.3.2") + (checkout / "README.md").write_text("# a later readme\n", encoding="utf-8") + _git(checkout, "add", "-A") + _git(checkout, "commit", "-q", "-m", "third release") + _git(checkout, "tag", "v0.3.3") + + tree = tmp_path / "mixed" / "source" + (tree / "emrg" / "server").mkdir(parents=True) + (tree / SHARED).write_text(RELEASED, encoding="utf-8") # v0.3.1's copy + (tree / "README.md").write_text("# a later readme\n", encoding="utf-8") # v0.3.3's copy + + done = _run(tree, checkout) + + assert done.returncode == 0, done.stdout + done.stderr + assert "no single release" in done.stdout, done.stdout + assert "newest release is v0.3.3" in done.stdout, done.stdout + + +def test_a_drift_verdict_still_carries_the_release_reading( + install_dir: Path, checkout: Path +) -> None: + """Both facts, one run: the tree lags *and* holds content no commit has. + + That is the historical incident's shape, and it is why the release line is printed under + both verdicts. A hand-edited tree matches no release, so it is named as such rather than + attributed to the one it is closest to - while the newest release is still named, because + that is the upgrade that replaces the edit. + """ + _git(checkout, "tag", "v0.3.1") + _recommit(checkout, "narrowed content\n", "narrow the rule") + _git(checkout, "tag", "v0.3.2") + (install_dir / "README.md").write_text("# edited in place\n", encoding="utf-8") + + done = _run(install_dir, checkout) + + assert done.returncode == 1, "content in no commit is drift: " + done.stdout + done.stderr + assert "hold content no shipped commit has" in done.stdout, done.stdout + assert "behind:" in done.stdout, ( + "a drift verdict must not swallow the release reading: " + done.stdout + ) + assert "no single release" in done.stdout and "newest release is v0.3.2" in done.stdout, ( + done.stdout + ) From fa07a3d3ee31cc11aa08a372e788458e17e81e89 Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Tue, 20 Oct 2026 10:37:33 +0800 Subject: [PATCH 2/2] emrg: an unreadable tag is a skipped release, and the remedy follows the direction The release reading measured two things correctly and answered a third wrongly, all three measured on a peer host (cycle cyc20261020-100652) and reproduced here before this change: - Two release tags in that clone answer "fatal: not a tree object" (their trees are absent, the same clone whose membership line already reports 40 absent objects). The walk raised on the first of them and main's "could not measure" took the drift verdict down with it: rc 2 where the reading before the branch answered 0. A tag this clone cannot read is one release it cannot name - skipped, counted and printed now, under both verdicts, the way the membership line counts what was not walked. - A tag list that stops short of the install tree is ordinary on a checkout nobody has fetched lately, and walking it as the only source answered "no single release" about a tree that is exactly a release - or named one three releases OLDER and recommended "an upgrade to v0.3.18" for a 0.3.21 tree. The install tree's own version file now names the tag to confirm first, so a stale list cannot decide what the tree is. - "The newest release is X" stood on a reading of tags while saying "release". The line now says "the newest release tag this clone carries", which is what was measured. - The remedy follows the direction: an upgrade is recommended only when the tree is demonstrably older; when it is newer the line says the tags lag and names no upgrade; when neither side could be ordered it says so and names no remedy at all. Two arms pin the failures the fixtures could not express before - a tag whose object is forged absent, and a tree committed but newer than every tag - and both are red on the head they answer. The double install-tree walk is gone as well: scan returns the shared list it walked instead of main walking it a second time. --- scripts/check-install-drift.py | 337 ++++++++++++++++++++++++++---- tests/test_check_install_drift.py | 124 ++++++++++- 2 files changed, 410 insertions(+), 51 deletions(-) diff --git a/scripts/check-install-drift.py b/scripts/check-install-drift.py index 1849f207..5059963d 100644 --- a/scripts/check-install-drift.py +++ b/scripts/check-install-drift.py @@ -128,6 +128,20 @@ * Objects absent from this clone are not walked, and the count is printed when it is not zero. The set is therefore the history this clone can see, which is the only history it can promise a file is safe in. +* **A release tag this clone cannot read is skipped and counted, never fatal.** Two tags on a + peer host answer `fatal: not a tree object`; raising on the first made the whole reading + `2`, which took the drift verdict down with the release question. `tags:` counts what was + skipped, the same way `membership:` counts what was not walked. +* **"Newest" is the newest release tag *this clone carries***, and the line says so. A tag + list that stops short is ordinary - a checkout nobody has fetched lately - and a sentence + about releases may not stand on a reading of tags. The install tree's own `__version__` + names the tag to confirm first, so a stale list cannot make the tree's release unnameable. +* **The direction is measured before a remedy is written.** An upgrade is recommended only + when the tree is demonstrably older than the newest readable tag; when the tree is newer + the line says the *tags* lag and names no upgrade, and when neither side could be ordered + it says the direction was not established and names no remedy at all. +* The fallback walk stops after `TAGS_EXAMINED` tags and says when it stopped, so a bounded + search is never read as "this tree matches no release". * It compares **content, under git's own line-ending convention**: a file whose only difference from the committed blob is CRLF versus LF reads as *no* drift, because `hash-object --path=` cleans it exactly as `git add` did. That is deliberate - such @@ -144,6 +158,7 @@ import subprocess import sys import traceback +from dataclasses import dataclass, field from pathlib import Path @@ -209,6 +224,50 @@ def tracked_files(root: Path) -> set[str]: #: "which release is this install tree" question may be answered from. RELEASE_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$") +#: How many release tags the fallback walk may read before it stops and says so. The walk is +#: a fallback, not the only source (see `behind_release`), and a clone carrying a hundred tags +#: is a hundred `ls-tree -r` calls on a run where nothing matches - measured 2026-10-20 on a +#: peer host where the walk reached tag 12 of 118 before dying on an unreadable one. +TAGS_EXAMINED = 20 + +#: Where the install tree states its own version. A **local** fact, and the reason it matters +#: is that it does not depend on this clone's tag list being complete or readable - the one +#: input a stale or holed tag walk gets wrong (see `behind_release`). +INSTALL_VERSION_PATH = "emrg/__init__.py" + +_VERSION_ASSIGNMENT_RE = re.compile(r"""__version__\s*=\s*["']([^"']+)["']""") +_VERSION_PARTS_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$") + + +def install_version(install_dir: Path) -> str | None: + """The version the install tree says it is, or `None` when it does not say. + + A claim about itself, not a reading of history: the file is `emrg/__init__.py`, which an + install carries and an upgrade replaces. `behind_release` uses it to pick the tag to + confirm, and names a release from it only when that tag's tree agrees on every shared + path, so a stale hand edit cannot borrow a release's name. + """ + path = install_dir / INSTALL_VERSION_PATH + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + match = _VERSION_ASSIGNMENT_RE.search(text) + return match.group(1) if match else None + + +def version_parts(text: str | None) -> tuple[int, int, int] | None: + """`"0.3.21"` → `(0, 3, 21)`, and `"v0.3.21"` → the same; `None` otherwise. + + Only used to order two versions, which a string comparison gets wrong at `0.3.10` against + `0.3.9` - the same defect the tag sort is `--sort=-v:refname` to avoid, one layer down. + """ + if not text: + return None + match = _VERSION_PARTS_RE.match(text.strip()) + return tuple(int(part) for part in match.groups()) if match else None + + def release_tags(root: Path) -> list[str]: """Every release tag this clone carries, newest first. @@ -222,15 +281,28 @@ def release_tags(root: Path) -> list[str]: return [name for name in (line.strip() for line in listing.splitlines()) if RELEASE_TAG_RE.match(name)] -def release_tree(root: Path, ref: str) -> dict[str, str]: - """`{path: blob id}` for one release ref, from a single `ls-tree`. +def release_tree(root: Path, ref: str) -> dict[str, str] | None: + """`{path: blob id}` for one release ref, or `None` when this clone cannot read it. One call for the whole tree rather than a `rev-parse` per path: the question compares every shared file at once, and this host's install tree shares ~380 of them, which at one process each is a second of subprocesses per release examined. + + `None` rather than an exception, because **a tag this clone cannot read is one release it + cannot name, not a reason to answer nothing**. Measured 2026-10-20 on a peer host + (`cyc20261020-100652`): two tags answered `fatal: not a tree object` - their trees are + absent from that clone, the same clone whose `membership:` line already reports 40 absent + objects - the walk raised on the first of them, and `main`'s `could not measure` took the + **drift verdict** down with it: `rc 2` where the reading before this one answered `0`. The + tolerance here is the one the membership walk already has, for the same reason: a narrowed + set is printed with its count, never traded for no reading at all. """ + try: + listing = _git(root, "ls-tree", "-r", ref) + except RuntimeError: + return None blobs: dict[str, str] = {} - for line in _git(root, "ls-tree", "-r", ref).splitlines(): + for line in listing.splitlines(): if "\t" not in line: continue meta, path = line.split("\t", 1) @@ -240,9 +312,32 @@ def release_tree(root: Path, ref: str) -> dict[str, str]: return blobs +@dataclass(frozen=True) +class ReleaseReading: + """What "which release is this tree, and is a newer one out" answered. + + `newest` is the newest release **tag this clone carries** - worded as a tag because that + is what was read, and a sentence about *releases* may not stand on a reading of tags + (measured 2026-10-20: a peer host's clone stops three releases short, so the newest tag + was `v0.3.18` while the tree was `0.3.21`). `tree_version` is the install tree's own + claim, `direction` orders the two so the remedy cannot point backwards, and + `unreadable` / `truncated` say how much of the tag list could be read at all. + """ + + newest: str | None = None + install_release: str | None = None + differing: list[str] = field(default_factory=list) + compared: int = 0 + tree_version: str | None = None + direction: str = "unknown" + unreadable: tuple[str, ...] = () + truncated: bool = False + + + def behind_release( install_dir: Path, root: Path, shared: list[tuple[Path, str]] -) -> tuple[str | None, str | None, list[str], int]: +) -> ReleaseReading: """`(newest tag, the tag this tree is, paths that differ from the newest, shared count)`. The question the rest of this tool cannot answer, and the reason it is here: `scan` asks @@ -263,6 +358,22 @@ def behind_release( tree's version is the newest tag whose content it matches on every shared path, which is one fact about one directory. + Three things the tag walk alone gets wrong, all measured 2026-10-20 on a peer host + (`cyc20261020-100652`) and each answered here: + + - **A tag this clone cannot read is skipped and counted**, never fatal - `release_tree` + returns `None` and the list lands in `unreadable`, which `main` prints. The drift verdict + is not collateral damage of the second question. + - **The tree's own version file names the tag to confirm first.** A tag list that stops + short of the install tree is ordinary on any checkout that has not fetched lately, and + walking it as the only source answers "no single release" about a tree that is exactly a + release - or, worse, names one *older* than the tree. `INSTALL_VERSION_PATH` is a local + fact, so it survives a stale list; the walk below is the fallback for an install that + does not carry the file. + - **The order of the two is established before a remedy is written.** `direction` is + `ahead` when the tree's own version is newer than the newest readable tag, and then no + upgrade is recommended: the tags are what lag. + A tree matching no tag is `None` and is not this reading's fault to explain: content in no tag is the *other* reading's subject (a hand edit, which destroys on upgrade), and mixing the two in one line leaves a reader unable to tell "an upgrade brings the rest" from "an @@ -271,33 +382,107 @@ def behind_release( Absent files are out of scope: this compares the paths both trees carry, so a file the newest release added is something an upgrade creates, not a lag in what this tree has. - `None` for the newest tag is "this clone carries no release tag", which the caller must - report as unmeasurable rather than as currency - `[]` with a tag is "measured, nothing lags". + `None` for the newest tag is "this clone carries no release tag it could read", which the + caller must report as unmeasurable rather than as currency - an empty `differing` with a + tag is "measured, nothing lags". """ if not shared: - return None, None, [], 0 + return ReleaseReading(compared=0) tags = release_tags(root) if not tags: - return None, None, [], len(shared) + return ReleaseReading(compared=len(shared)) ids = {relative: _git_stored_id(root, path, relative) for path, relative in shared} - # The tree's own version: the newest release it matches on every shared path. Walked from - # the newest so the first match is the answer, and the walk stops there. - install_release: str | None = None - for tag in tags: + tree_version = install_version(install_dir) + unreadable: list[str] = [] + trees: dict[str, dict[str, str]] = {} + failed: set[str] = set() + + def tree_of(tag: str) -> dict[str, str] | None: + """One tag's blobs, read once; `None` for a tag this clone cannot read. + + A failure is remembered as well as a success: the list below reads some tags twice + (once to find the newest readable one, once as the fallback walk), and a skipped tag + that is retried is counted twice in the note that reports it - measured on the arm + below before this cache existed, which read `2 release tag(s) ... (v0.3.9, v0.3.9)`. + """ + if tag in trees: + return trees[tag] + if tag in failed: + return None blobs = release_tree(root, tag) - if all(blobs.get(relative) == blob for relative, blob in ids.items()): - install_release = tag + if blobs is None: + failed.add(tag) + unreadable.append(tag) + return None + trees[tag] = blobs + return blobs + + def matches(tag: str) -> bool: + blobs = tree_of(tag) + return blobs is not None and all( + blobs.get(relative) == blob for relative, blob in ids.items() + ) + + # The newest tag this clone can actually read: the first readable one in version order, + # since `release_tags` sorted them. An unreadable newest tag is a release this reading + # cannot name, so it is skipped and counted rather than reported as the answer. + newest: str | None = None + for tag in tags: + if tree_of(tag) is not None: + newest = tag break + if newest is None: + return ReleaseReading( + compared=len(shared), + tree_version=tree_version, + unreadable=tuple(unreadable), + truncated=len(tags) > TAGS_EXAMINED, + ) - newest = tags[0] - newest_blobs = release_tree(root, newest) + # The tree's own version names the tag to confirm first - one `ls-tree` - and the bounded + # walk is the fallback, so a stale list cannot decide what the tree is. + install_release: str | None = None + candidate = f"v{tree_version}" if tree_version else None + if candidate and candidate in tags and matches(candidate): + install_release = candidate + else: + for tag in tags[:TAGS_EXAMINED]: + if matches(tag): + install_release = tag + break + + newest_blobs = tree_of(newest) or {} differing = sorted( relative for relative, blob in ids.items() if newest_blobs.get(relative) != blob ) - return newest, install_release, differing, len(shared) + + direction = "current" + if differing: + tree_parts = version_parts(tree_version) or version_parts(install_release) + newest_parts = version_parts(newest) + if tree_parts and newest_parts and tree_parts > newest_parts: + direction = "ahead" + elif tree_parts and newest_parts and tree_parts < newest_parts: + direction = "behind" + else: + # No version on either side to order: the difference is real and the direction is + # not measured, which is what the report says rather than naming a release. + direction = "unknown" + + return ReleaseReading( + newest=newest, + install_release=install_release, + differing=differing, + compared=len(shared), + tree_version=tree_version, + direction=direction, + unreadable=tuple(unreadable), + truncated=install_release is None and len(tags) > TAGS_EXAMINED, + ) + #: The ref namespaces a release is built from. Everything else a ref can live in @@ -400,14 +585,7 @@ def shared_paths(install_dir: Path, root: Path) -> tuple[list[tuple[Path, str]], BEHIND_SHOWN = 10 -def _report_release( - install_dir: Path, - root: Path, - newest: str | None, - install_release: str | None, - differing: list[str], - checked: int, -) -> None: +def _report_release(install_dir: Path, reading: ReleaseReading) -> None: """Print which release this tree is, and whether a newer one exists. Printed under **both** verdicts, because it is a second question and not a verdict: a tree @@ -415,41 +593,86 @@ def _report_release( it). Leaving it out of the green branch is the defect this function exists to fix - the green line said "a release's refs still reach this content", which is true of any older release and therefore said nothing about being current. + + **The remedy follows the direction, or there is none.** "An upgrade to X brings these" is + true only when X is newer than the tree, and measured 2026-10-20 on a peer host it was + printed for a tree three releases *newer* than X - advice to downgrade, from a reading that + had never established which side was ahead. So the direction is stated, and when it was not + measured the line says that instead of naming a release. """ - if newest is None: + if reading.newest is None: + if reading.unreadable: + shown = ", ".join(reading.unreadable[:5]) + more = ( + f" and {len(reading.unreadable) - 5} more" + if len(reading.unreadable) > 5 + else "" + ) + print( + f"behind: could not measure which release {install_dir} is - this checkout " + f"carries {len(reading.unreadable)} release tag(s) and none of them can be " + f"read here ({shown}{more})" + ) + return print( f"behind: could not measure which release {install_dir} is - this checkout " "carries no release tag, and the refs a release is built from cannot answer it " "(an older release's bytes are in them too)" ) return - if not differing: + if not reading.differing: print( - f"behind: none - {install_dir} carries what {newest} ships " - f"({checked} shared file(s) compared)" + f"behind: none - {install_dir} carries what {reading.newest} ships " + f"({reading.compared} shared file(s) compared)" ) return - version = install_release or "no single release" + version = reading.install_release or "no single release" print( - f"behind: {install_dir} is the {version} copy and the newest release is {newest}; " - f"{len(differing)} of {checked} shared file(s) differ, so whatever the daemon renders " - "from this tree is that release's copy" + f"behind: {install_dir} is the {version} copy and the newest release tag this " + f"clone carries is {reading.newest}; {len(reading.differing)} of {reading.compared} " + "shared file(s) differ, so whatever the daemon renders from this tree is that " + "release's copy" ) - for relative in differing[:BEHIND_SHOWN]: + for relative in reading.differing[:BEHIND_SHOWN]: print(f" {relative}") - if len(differing) > BEHIND_SHOWN: - print(f" ... and {len(differing) - BEHIND_SHOWN} more") + if len(reading.differing) > BEHIND_SHOWN: + print(f" ... and {len(reading.differing) - BEHIND_SHOWN} more") + if reading.direction == "ahead": + print( + f" direction: this tree reports {reading.tree_version or 'no version'}, which " + f"is newer than {reading.newest} - so nothing here is behind; the tags this " + f"clone carries are. An upgrade to {reading.newest} would move this tree " + "backwards." + ) + print( + " remedy: fetch the tags in the source checkout (`git fetch --tags`), so the " + "release this tree is can be named and an upgrade can be compared against it." + ) + return + if reading.direction == "behind": + print( + f" remedy: an upgrade to {reading.newest} brings these. Until then a rule read " + "from this tree is the installed release's rule, not the newest one's." + ) + return print( - f" remedy: an upgrade to {newest} brings these. Until then a rule read from this " - "tree is the installed release's rule, not the newest one's." + " direction: not established - neither this tree's own version nor a tag it " + "matches could be read, so whether an upgrade to " + f"{reading.newest} moves it forward or backwards is not measured, and no remedy is " + "named." ) + def scan( install_dir: Path, root: Path -) -> tuple[list[tuple[str, int, str]], list[str], int, int]: - """Return `(drifted, skipped, checked, missing)` for the files the two trees share. +) -> tuple[list[tuple[str, int, str]], list[str], int, int, list[tuple[Path, str]]]: + """Return `(drifted, skipped, checked, missing, shared)` for the paths the trees share. + + `shared` is returned rather than recomputed: the release reading starts from the same + walk, and calling `shared_paths` twice walked the whole install tree twice on every run + (`cyc20261020-092940`'s nit on this branch). `drifted` is `(path, size, note)` for a shared file whose content no shipped object holds - the note saying why, and naming the bookkeeping ref it is still recoverable @@ -485,7 +708,7 @@ def scan( else: note = "no commit of this checkout holds this content" reported.append((relative, size, note)) - return reported, skipped, len(shared), missing + return reported, skipped, len(shared), missing, shared def _entry() -> int: @@ -568,9 +791,8 @@ def main(argv: list[str] | None = None) -> int: ) return 2 try: - drifted, skipped, checked, missing = scan(install_dir, root) - shared, _ = shared_paths(install_dir, root) - newest, install_release, differing, _ = behind_release(install_dir, root, shared) + drifted, skipped, checked, missing, shared = scan(install_dir, root) + reading = behind_release(install_dir, root, shared) except RuntimeError as exc: print(f"could not measure: {exc}", file=sys.stderr) return 2 @@ -585,6 +807,29 @@ def main(argv: list[str] | None = None) -> int: "missing tree reads as drift" ) + if reading.unreadable: + # Same rule one layer over: a tag whose objects are absent names no tree here, so it + # is one release this reading cannot name - and the count is what stops the named + # ones being read as the whole list. Printed under both verdicts because it is a + # property of the reading, not of the files. + shown = ", ".join(reading.unreadable[:5]) + more = ( + f" and {len(reading.unreadable) - 5} more" + if len(reading.unreadable) > 5 + else "" + ) + print( + f"tags: {len(reading.unreadable)} release tag(s) this checkout carries cannot be " + f"read here ({shown}{more}) and were skipped; each is one release this reading " + "cannot name" + ) + if reading.truncated: + print( + f"tags: the walk stopped after {TAGS_EXAMINED} release tag(s), so this tree " + "matched none of those - that is a bounded search, not a measurement that it " + "matches no release" + ) + if checked == 0: # The family's empty-set refusal (2026-10-06, `cyc20261006-165503`): a verdict of # `0` says the files were checked, and a tree sharing no path with this checkout @@ -638,7 +883,7 @@ def main(argv: list[str] | None = None) -> int: f"{len(leftover)} named as leftovers above, " f"{len(caches)} interpreter cache file(s) not listed)" ) - _report_release(install_dir, root, newest, install_release, differing, checked) + _report_release(install_dir, reading) return 0 @@ -650,7 +895,7 @@ def main(argv: list[str] | None = None) -> int: "install/upgrade replaces it whole, so the next one destroys it: move the change " "into the source checkout and ship it, or it is lost." ) - _report_release(install_dir, root, newest, install_release, differing, checked) + _report_release(install_dir, reading) return 1 diff --git a/tests/test_check_install_drift.py b/tests/test_check_install_drift.py index e166c058..a4d83257 100644 --- a/tests/test_check_install_drift.py +++ b/tests/test_check_install_drift.py @@ -756,7 +756,8 @@ def test_a_tree_that_is_an_older_release_is_named_as_behind( assert done.returncode == 0, done.stdout + done.stderr assert "still reach" in done.stdout, "the drift reading is unchanged: " + done.stdout assert "behind:" in done.stdout, done.stdout - assert "v0.3.1 copy" in done.stdout and "newest release is v0.3.2" in done.stdout, ( + assert "v0.3.1 copy" in done.stdout, done.stdout + assert "newest release tag this clone carries is v0.3.2" in done.stdout, ( "the line must name the tree's own release and the newest one: " + done.stdout ) assert SHARED in done.stdout, ( @@ -767,6 +768,10 @@ def test_a_tree_that_is_an_older_release_is_named_as_behind( "the size of the gap is a derived number, so it is asserted rather than left to a " "reader to infer from a path list: " + done.stdout ) + assert "remedy: an upgrade to v0.3.2 brings these" in done.stdout, ( + "the remedy is only true in this direction, so the arm that establishes the " + "direction pins it: " + done.stdout + ) def test_a_tree_that_is_the_newest_release_is_not_reported_as_behind( @@ -784,7 +789,7 @@ def test_a_tree_that_is_the_newest_release_is_not_reported_as_behind( assert done.returncode == 0, done.stdout + done.stderr assert "behind: none" in done.stdout, done.stdout assert "carries what v0.3.1 ships" in done.stdout, done.stdout - assert "copy and the newest release" not in done.stdout, done.stdout + assert "copy and the newest release tag" not in done.stdout, done.stdout def test_a_clone_with_no_release_tag_says_it_could_not_measure( @@ -821,7 +826,7 @@ def test_the_newest_release_is_read_in_version_order_not_lexically( done = _run(install_dir, checkout) assert done.returncode == 0, done.stdout + done.stderr - assert "newest release is v0.3.10" in done.stdout, done.stdout + assert "newest release tag this clone carries is v0.3.10" in done.stdout, done.stdout assert "v0.3.9 copy" in done.stdout, done.stdout @@ -848,7 +853,7 @@ def test_a_tree_matching_no_release_names_no_version(tmp_path: Path, checkout: P assert done.returncode == 0, done.stdout + done.stderr assert "no single release" in done.stdout, done.stdout - assert "newest release is v0.3.3" in done.stdout, done.stdout + assert "newest release tag this clone carries is v0.3.3" in done.stdout, done.stdout def test_a_drift_verdict_still_carries_the_release_reading( @@ -873,6 +878,115 @@ def test_a_drift_verdict_still_carries_the_release_reading( assert "behind:" in done.stdout, ( "a drift verdict must not swallow the release reading: " + done.stdout ) - assert "no single release" in done.stdout and "newest release is v0.3.2" in done.stdout, ( + assert "no single release" in done.stdout, done.stdout + assert "newest release tag this clone carries is v0.3.2" in done.stdout, ( done.stdout ) + + +def test_a_release_tag_this_clone_cannot_read_is_skipped_not_fatal( + tmp_path: Path, install_dir: Path, checkout: Path +) -> None: + """One unreadable tag must not take the drift verdict down with it. + + Measured 2026-10-20 on a peer host (`cyc20261020-100652`): two release tags answer + `fatal: not a tree object` - their trees are absent from that clone, the same clone whose + `membership:` line already reports 40 absent objects - and the release walk raised on the + first of them, so `main` answered `could not measure` and **`2`** where the reading before + it answered `0`. A second question may not cost the first one's answer. + + The hole is forged rather than waited for, as the absent-object arm does for membership: a + tag ref is written pointing at an object this clone does not have. The forgery is checked + before the assertion, so a fixture that fails to create the condition cannot pass. + """ + _git(checkout, "tag", "v0.3.1") + + ref = checkout / ".git" / "refs" / "tags" / "v0.3.9" + ref.parent.mkdir(parents=True, exist_ok=True) + ref.write_text("0" * 39 + "1\n", encoding="utf-8") + + forged = subprocess.run( + ["git", "-C", str(checkout), "ls-tree", "-r", "v0.3.9"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + assert forged.returncode != 0, ( + "the forgery did not create the condition: the tag still reads, so this case would " + "pass with or without the tolerance" + ) + + done = _run(install_dir, checkout) + + assert done.returncode == 0, ( + "an unreadable tag is one release this reading cannot name, not a reason to answer " + f"nothing: {done.stdout!r} {done.stderr!r}" + ) + assert "still reach" in done.stdout, ( + "the drift verdict is the reading's reason to exist and must survive: " + done.stdout + ) + assert "tags: 1 release tag(s)" in done.stdout and "v0.3.9" in done.stdout, ( + "the skipped tag has to be counted and named, or a narrowed list reads as the whole " + "one: " + done.stdout + ) + assert "behind: none" in done.stdout, ( + "the newest tag that *can* be read is still v0.3.1 and the tree is its copy, so the " + "verdict over what was readable is unchanged: " + done.stdout + ) + + +def test_a_tree_newer_than_every_release_tag_is_not_reported_as_behind( + tmp_path: Path, checkout: Path +) -> None: + """The direction, not just the distance: advice to downgrade is the defect. + + A checkout nobody has fetched lately carries tags that stop short of the install tree - + ordinary, and measured 2026-10-20 on a peer host where the newest tag was `v0.3.18` and + the tree reported `0.3.21`. The line printed there named `v0.3.18` as "the newest release" + and recommended "an upgrade to v0.3.18", for a tree three releases **newer**. The tree's + own version file is what answers it here, and the remedy has to follow. + """ + (checkout / "emrg" / "__init__.py").write_text( + '__version__ = "0.3.1"\n', encoding="utf-8" + ) + _git(checkout, "add", "-A") + _git(checkout, "commit", "-q", "-m", "the version file ships too") + _git(checkout, "tag", "v0.3.1") + + # Two releases of work the tag list does not know about - committed, so the drift + # reading is green and the case is about the release line alone. + _recommit(checkout, "narrowed content\n", "the next release's content") + (checkout / "emrg" / "__init__.py").write_text( + '__version__ = "0.3.3"\n', encoding="utf-8" + ) + _git(checkout, "add", "-A") + _git(checkout, "commit", "-q", "-m", "version 0.3.3") + + tree = tmp_path / "install" / "source" + (tree / "emrg" / "server").mkdir(parents=True) + for name in (SHARED, "README.md", "emrg/__init__.py"): + shutil.copyfile(checkout / name, tree / name) + + done = _run(tree, checkout) + + assert done.returncode == 0, done.stdout + done.stderr + assert "still reach" in done.stdout, ( + "the content is committed, so the drift reading is green and this case is about the " + "release line: " + done.stdout + ) + assert "newer than v0.3.1" in done.stdout, ( + "the line has to say which side is ahead, or the remedy is a guess: " + done.stdout + ) + assert "0.3.3" in done.stdout, ( + "the tree's own version is the local fact that answers this: " + done.stdout + ) + assert "would move this tree backwards" in done.stdout, ( + "an upgrade to an older tag is the advice that made this a defect: " + done.stdout + ) + assert "remedy: an upgrade to v0.3.1 brings these" not in done.stdout, ( + "the remedy must not recommend a downgrade: " + done.stdout + ) + assert "git fetch --tags" in done.stdout, ( + "what is actually wrong is the tag list, and the line must say so: " + done.stdout + )