From 385baacebfe548bd5fa4cc289e0798d36ff7a1ee Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 02:40:38 +0530 Subject: [PATCH 01/37] feat: add read-only release state and PR title lint Adds the release tooling under .github/release and two workflows that write nothing: state.yaml prints every release/X.Y branch with its state, and lint-pr.yaml checks PR titles against the release rule. The OSS line reads its own ee-vX.Y.Z git tags; no registry is read. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 1047 +++++++++++++++++++++++ .github/release/backport_merge.py | 359 ++++++++ .github/release/convgate.py | 531 ++++++++++++ .github/release/oss.py | 361 ++++++++ .github/release/relmgr.py | 634 ++++++++++++++ .github/release/relstate.py | 578 +++++++++++++ .github/release/ruff.toml | 7 + .github/release/test_backport.py | 927 ++++++++++++++++++++ .github/release/test_backport_merge.py | 404 +++++++++ .github/release/test_convgate.py | 946 +++++++++++++++++++++ .github/release/test_links.py | 43 + .github/release/test_oss.py | 973 +++++++++++++++++++++ .github/release/test_relstate.py | 1085 ++++++++++++++++++++++++ .github/workflows/lint-pr.yaml | 69 ++ .github/workflows/state.yaml | 36 + 15 files changed, 8000 insertions(+) create mode 100644 .github/release/backport.py create mode 100644 .github/release/backport_merge.py create mode 100644 .github/release/convgate.py create mode 100644 .github/release/oss.py create mode 100644 .github/release/relmgr.py create mode 100644 .github/release/relstate.py create mode 100644 .github/release/ruff.toml create mode 100644 .github/release/test_backport.py create mode 100644 .github/release/test_backport_merge.py create mode 100644 .github/release/test_convgate.py create mode 100644 .github/release/test_links.py create mode 100644 .github/release/test_oss.py create mode 100644 .github/release/test_relstate.py create mode 100644 .github/workflows/lint-pr.yaml create mode 100644 .github/workflows/state.yaml diff --git a/.github/release/backport.py b/.github/release/backport.py new file mode 100644 index 0000000000..be27dc998e --- /dev/null +++ b/.github/release/backport.py @@ -0,0 +1,1047 @@ +"""Backport: a fix lands on main first, then is cherry-picked with -x onto every release branch that +accepts it, all at once. + +Usage: backport.py --state state.json (--commits SHA|PR[,...] | --rescan) [--confirm-stabilized] [--branches release/X.Y[,...]] + +--rescan (every push to main): ignore the pushed commits and pick, for each target, every main commit since its cut +that no pick or decline trailer there names: plain fixes too on fix targets (held accept=OWED), security/critical +only on stabilized ones (accept=HELD, held for the RM without --confirm-stabilized). A run cancelled by a newer push +costs nothing: the next run rescans. A decline carries the pick trailer, so a declined fix is never re-picked; an +open conflict PR is left alone (`skipped: conflict PR pending`) until a person merges or closes it; an open clean +pick PR whose merge failed in an earlier run is merged by the next run. +--commits (dispatch): only these commits. + +Run from a full clone whose `origin` pushes with the bot token, with GH_TOKEN set to the bot token. +Every pick is a PR from backport/X.Y/ to release/X.Y titled with the original main PR's title (every pick +is its own PR), so PR-title changelogs list it. The class comes from that PR's title and fix/ labels. +Clean pick: the bot squash-merges the PR in the same run through backport_merge.squash with the pick's own +message, so the -x trailer survives whatever the repo's squash default, and deletes +the head. No wait for the PR check: the bot is a bypass actor on the release/* ruleset, the check only +classifies commits, which an -x pick passes, and the push run of Release branch CI stays the backstop, as for +the direct pushes this replaced. If GitHub still holds the merge for a required check, the PR stays open and +the run reports it. +A rejected push of the pick head is retried (3 attempts, 5 s then 20 s apart); a final rejection is one +sentence naming the cause and the fix, in the run's ::error::, on the source PR and in the gate's MISSING row. +Conflict: committed with markers and left open with label backport-conflict. A person resolves it, gets one +approving review and merges it, or closes it; the Backport PR workflow (backport_merge.py) then records the merge, +or the close as a decline (the close comment is the reason). Reopening and merging undoes a decline. +Never builds. Targets come from relstate's backport_targets only, never from input: a branch is a +target when its state accepts the fix. --branches only narrows those targets; it never adds one. +""" + +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import time +import traceback + +import backport_merge as bm +import relmgr as rm +import relstate as rs + +REPO = "." +BOT_ENV = {"GIT_COMMITTER_NAME": "releases-bot", "GIT_COMMITTER_EMAIL": rm.BOT_EMAIL} + + +def run(*args, check=True, input=None, env=None): + r = subprocess.run( + args, + capture_output=True, + text=True, + input=input, + env={**os.environ, **BOT_ENV, **(env or {})}, + ) + if check and r.returncode: + raise RuntimeError( + f"{' '.join(args[:3])} (exit {r.returncode}): {r.stderr.strip()}" + ) + return r + + +def soft(what, *args): + """A best-effort command (a comment, a head delete): a failure is a ::warning::, never the run's.""" + r = run(*args, check=False) + if r.returncode: + print(f"::warning::{what} failed (exit {r.returncode}): {r.stderr.strip()}") + return r + + +def gh(*args, token=None): + """Gh as GH_TOKEN, or as token (another repo's) when given.""" + return run("gh", *args, env={"GH_TOKEN": token} if token else None).stdout.strip() + + +def conflict_prs(slug, base=None, token=None): + """Open backport-conflict PRs of repo slug, [{number, url, baseRefName, createdAt}], only those on base when given. + A failed listing raises: it never reads as no PRs. + """ + prs = json.loads( + gh( + "pr", + "list", + "-R", + slug, + "--label", + "backport-conflict", + "--state", + "open", + "--json", + "number,url,baseRefName,createdAt", + "--limit", + "200", + token=token, + ) + ) + return [p for p in prs if base is None or p["baseRefName"] == base] + + +def open_conflicts(lines, annotate="warning"): + """lines: [(slug, branch, token)]. One ::annotate per open backport-conflict PR on each line, then + (refs like `slug#n`, one summary line or None). A failed listing raises. + """ + refs, links = [], [] + for slug, branch, token in lines: + for p in conflict_prs(slug, branch, token): + ref = f"{slug}#{p['number']}" + print( + f"::{annotate}::Open backport conflict PR {ref} on {branch}: resolve and merge it, or close it to decline" + ) + refs.append(ref) + links.append(f"[{ref}]({p['url']}) on `{branch}`") + return refs, (f"⚠ Open backport conflict PRs: {', '.join(links)}" if links else None) + + +def resolve(slug, item): + """A main SHA or PR number (`#` optional) -> full SHA on origin/main; None if it is not on main.""" + item = item.removeprefix("#") # the summary's paste command names PRs as #n + if re.fullmatch( + r"\d{1,6}", item + ): # PR numbers are short; SHAs given here are 7+ hex chars + pr = json.loads(gh("api", f"repos/{slug}/pulls/{item}")) + if not pr["merged"]: + return None + item = pr["merge_commit_sha"] + r = rm.git(REPO, "rev-parse", "-q", "--verify", f"{item}^{{commit}}", check=False) + sha = r.stdout.strip() + return sha if r.returncode == 0 and rm.is_ancestor(REPO, sha, "origin/main") else None + + +def main_order(repo, shas): + """Shas oldest first in main's history, duplicates dropped, whatever order the caller gave.""" + return rm.git(repo, "rev-list", "--reverse", "--no-walk=sorted", *shas).split() + + +def fix_class(slug, sha, token=None): + """(class, PR ref, PR title) from the PR that merged sha, via relmgr.pr_fix_class (same rule as Lint PR). + token: another repo's, as in gh(). + """ + try: + out = gh("api", f"repos/{slug}/commits/{sha}/pulls", token=token) + except RuntimeError: # a read, and a rescan makes dozens: retry a transient error once ("unexpected end of JSON input") + out = gh("api", f"repos/{slug}/commits/{sha}/pulls", token=token) + prs = json.loads(out) + if not prs: + return None, "no PR", None + pr = prs[0] + return ( + rm.pr_fix_class(pr["title"], [label["name"] for label in pr["labels"]]), + f"#{pr['number']}", + pr["title"], + ) + + +def pr_body(sha, src, branch, head, files=()): + """Body of the pick PR; files = conflicted paths (empty for a clean pick).""" + origin = f"Backport of {src} (`{sha}`) to `{branch}`" + if not files: + return f"{origin}. Clean pick; the bot squash-merges it in the same Backport run." + return ( + f"{origin} conflicted. The conflicted tree is committed **with markers** in:\n\n" + + "".join(f"- `{p}`\n" for p in files) + + f"\nResolve on this branch (`{head}`): amend the pick commit or push a fix-up commit; the " + f"first commit's `(cherry picked from commit {sha})` line carries into the squash. Conflict markers " + "left in any file fail the PR check. Resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." + ) + + +def only_listed(targets, listed): + """(targets kept, listed branches that are not targets); an empty listed keeps every target.""" + if not listed: + return targets, [] + return [b for b in targets if b in listed], [b for b in listed if b not in targets] + + +def held_line(sha, cls, pr, branch, annotate="warning", slug=None): + """slug: the other repo the fix is from, named before its PR.""" + return f"::{annotate}::{sha[:7]} ({cls}, PR {slug or ''}{pr}) held for {branch}: stabilized, needs confirm_stabilized" + + +def held_warnings(awaiting): + """One warning per stabilized branch (first-seen order) for awaiting [(sha, branch, class, PR ref)].""" + counts = {} + for _, b, *_ in awaiting: + counts[b] = counts.get(b, 0) + 1 + return [ + f"::warning::{b}: {n} held security/critical fix{'es' if n > 1 else ''}, Backport with confirm_stabilized" + for b, n in counts.items() + ] + + +STALE_DAYS = 5 # a pending item this many days old gets a stale mark + + +def link_pr(ref): + """A `#n` PR ref as a link; anything else ("no PR", "-") as is.""" + return rm.pr_link(int(ref[1:])) if re.fullmatch(r"#\d+", ref) else ref + + +def latest_tag(repo, branch): + """(newest tag on the line, RC or stable, commits on the branch after it), or (None, None) before any. + A stable tag ranks above the RC it was promoted from (same commit, same patch). + """ + line = branch.split("/", 1)[1] + tags = [ + t + for t in rm.git( + repo, "tag", "--merged", f"origin/{branch}", "-l", f"{rs.PREFIX}v{line}.*" + ).split() + if (m := rs.VER_RE.fullmatch(t)) and f"{m[1]}.{m[2]}" == line + ] + if not tags: + return None, None + m = lambda t: rs.VER_RE.fullmatch(t) + tag = max(tags, key=lambda t: (int(m(t)[3]), m(t)[4] is None, int(m(t)[4] or 0))) + return tag, int(rm.git(repo, "rev-list", "--count", f"{tag}..origin/{branch}")) + + +def next_step(conflicts, held_n, tag, unbuilt): + """The one thing to do next on a line; first match wins.""" + if conflicts: + return f"resolve {rm.pr_link(conflicts[0])}" + if held_n: + return f"RM: confirm {held_n} security fix" + ("es" if held_n > 1 else "") + driver = ( + "cloud " if rs.PREFIX else "" + ) # prefixed (OSS) lines are built and promoted by the cloud workflows + if unbuilt: + return f"{driver}Take RC" + if tag and "-rc." in tag: + return f"{driver}Promote" + return "-" + + +def status_row(branch, state, tag, unbuilt, missing, conflicts, held_n): + """One `| line | state | latest tag | missing | pending | next step |` row; conflicts: open conflict PR numbers.""" + built = ( + "none" + if tag is None + else f"{rm.tag_link(tag)} · " + + (f"+{unbuilt} not built" if unbuilt else "all built") + ) + pending = ", ".join( + ([f"{len(conflicts)} conflict"] if conflicts else []) + + ([f"{held_n} RM"] if held_n else []) + ) + return ( + f"| `{branch}` | {state} | {built} | {missing} | {pending or '-'} | " + f"{next_step(conflicts, held_n, tag, unbuilt)} |" + ) + + +def this_run(rows): + """Only what this run did: merged picks, new conflict PRs, failures (skipped and invalid rows are left out).""" + out = [] + for sha, pr, _, b, res, _ in rows: + src = link_pr(pr) if pr.startswith("#") else rm.commit_link(sha) + if m := re.fullmatch(r"merged (\w+) \(#(\d+)\)", res): + out.append( + f"- {src} → `{b}`: merged as {rm.commit_link(m[1])} ({rm.pr_link(int(m[2]))})" + ) + elif res.startswith("pr "): + out.append( + f"- {src} → `{b}`: conflict, {rm.pr_link(int(res.rsplit('/', 1)[1]))} to resolve" + ) + elif res.startswith("failed"): + out.append(f"- {src} → `{b}`: {res}") + return out + + +def stale(days): + return f"{days} day{'s' if days != 1 else ''}" + (" ⚠" if days >= STALE_DAYS else "") + + +def pending_block(conflicts, held_rows): + """Folded detail. conflicts: [(n, base, days open)]; held_rows: [(sha, branch, class, PR ref, days since the + main merge)] for stabilized lines, grouped per commit, then one command that picks them all. + """ + groups = {} + for sha, b, cls, pr, days in held_rows: + groups.setdefault(sha, [cls, pr, days, []])[3].append(b) + n = len(conflicts) + len(groups) + if not n: + return [] + out = [f"
Pending ({n})", ""] + if conflicts: + out += [ + "**Conflict PRs** (resolve and merge, or close with `Declined: `):", + "", + ] + out += [ + f"- {rm.pr_link(c)} on `{b}` · open {stale(d)}" for c, b, d in conflicts + ] + [""] + if groups: + out += [ + "**Held for the RM** (stabilized lines; pick with `confirm_stabilized`):", + "", + ] + out += [ + f"- {link_pr(pr)} {cls} → " + + ", ".join(f"`{b}`" for b in bs) + + f" · merged {stale(d)} ago" + for cls, pr, d, bs in groups.values() + ] + commits = ",".join( + pr if re.fullmatch(r"#\d+", pr) else sha[:12] + for sha, (_, pr, _, _) in groups.items() + ) + branches = ",".join( + dict.fromkeys(b for _, _, _, bs in groups.values() for b in bs) + ) + out += [ + "", + "```", + f"gh workflow run backport.yaml -f commits={commits} -f branches={branches} " + "-f confirm_stabilized=true", + "```", + "", + ] + return out + ["
"] + + +def push_summary(head, table, done, pending): + """Push mode summary lines; a quiet push (nothing done, nothing pending) is the head line alone.""" + if not done and not pending: + return [f"{head} · nothing picked, nothing pending"] + out = [ + f"### {head}", + "", + "| line | state | latest tag | missing | pending | next step |", + "|---|---|---|---|---|---|", + *table, + "", + ] + out += ( + ["**This run**", "", *done, ""] if done else ["**This run:** nothing picked", ""] + ) + return out + pending + + +HELD = {"security", "critical"} +OWED = {"fix", "security", "critical"} +UNCLASSIFIABLE = "UNCLASSIFIABLE" # no PR found, or a title Lint PR should have refused: shown, never dropped + + +def held_filter(commits, classes, trailers, accept=HELD): + """Pure core of held(): the main commits whose class is in accept (or UNCLASSIFIABLE) and that no pick or + decline trailer on the branch names (trailers may be abbreviated). classes: {sha: (class, PR ref, ...)}. + """ + return [ + (s, *classes[s]) + for s in commits + if classes[s][0] in accept | {UNCLASSIFIABLE} + and not any(s.startswith(t) for t in trailers) + ] + + +def classes_from_listing(prs): + """Pure: {merge sha: (class, PR ref, title)} from `gh pr list --json number,title,labels,mergeCommit` rows; a title + Lint PR should have refused maps to UNCLASSIFIABLE with the error, as in held(). + """ + out = {} + for pr in prs: + if not pr.get("mergeCommit"): + continue + try: + cls = rm.pr_fix_class(pr["title"], [label["name"] for label in pr["labels"]]) + out[pr["mergeCommit"]["oid"]] = (cls, f"#{pr['number']}", pr["title"]) + except ValueError as e: + out[pr["mergeCommit"]["oid"]] = (UNCLASSIFIABLE, str(e), pr["title"]) + return out + + +def pr_classes(slug, since): + """{merge sha: (class, PR ref, title)} for every PR merged into main since `since` (YYYY-MM-DD): one search + listing per run, labels read now, so a label edited after the merge counts. A failed listing returns {}: + held() then looks each commit up on its own. + """ + # ponytail: the search API stops at 1000 results; a range with more merged PRs falls back per commit beyond them. + try: + out = gh( + "pr", + "list", + "-R", + slug, + "--state", + "merged", + "--base", + "main", + "--search", + f"merged:>={since}", + "--limit", + "1000", + "--json", + "number,title,labels,mergeCommit", + ) + except RuntimeError as e: + print(f"::warning::PR listing failed, classing commit by commit: {e}") + return {} + return classes_from_listing(json.loads(out)) + + +def held(slug, repo, branch, accept=HELD, cache=None, token=None): + """[(sha, class, PR ref, title)] of main commits since branch's cut whose class is in accept (or + UNCLASSIFIABLE), neither picked nor declined: the held (or, with OWED, owed) picks the RM must see. + cache: {sha: (class, PR ref, title)} shared across calls (the rescan primes it from pr_classes), so a main commit + is looked up on its own at most once per run: only when the listing missed it (no PR, search index lag). + token: slug's, when slug is not this repo (the OSS line). + """ + remote = f"origin/{branch}" + base = rm.git(repo, "merge-base", "origin/main", remote) + out = rm.git(repo, "rev-list", "--reverse", f"{base}..origin/main") + commits = out.split() if out else [] + classes = {} if cache is None else cache + looked = 0 + for s in commits: + if s in classes: + continue + looked += 1 + try: + cls, pr, title = fix_class(slug, s, token=token) + classes[s] = ( + (UNCLASSIFIABLE, pr, rm.git(repo, "log", "-1", "--format=%s", s)) + if pr == "no PR" + else (cls, pr, title) + ) + except ValueError as e: # a title Lint PR should have stopped + classes[s] = ( + UNCLASSIFIABLE, + str(e), + rm.git(repo, "log", "-1", "--format=%s", s), + ) + print( + f"held: {len(commits)} main commit(s) since the cut of {branch}, {looked} fix_class lookup(s)" + ) + # A decline carries the same trailer as a pick, so one grep covers both. + trailers = { + o + for c in rm.git( + repo, + "log", + "--format=%H", + "-F", + "--grep=(cherry picked from commit", + f"{base}..{remote}", + ).split() + for o in rm.picked_from(repo, c) + } + return held_filter(commits, classes, trailers, accept) + + +def owed_summary(rows, branch): + """Markdown: the owed-picks table for a branch gaining a late target, plus the line to paste into Backport's + commits input (UNCLASSIFIABLE rows are listed but left out of it: Backport cannot class them either). + """ + out = [ + f"**Owed picks** for `{branch}`: main fixes since the cut, neither picked nor declined", + "", + ] + if not rows: + return "\n".join(out + ["none"]) + out += ["| commit | PR | title | class |", "|---|---|---|---|"] + cell = lambda t: t.replace("|", r"\|") # a pipe in a title would split the row + out += [f"| `{s[:7]}` | {pr} | {cell(title)} | {cls} |" for s, cls, pr, title in rows] + return "\n".join( + out + + [ + "", + "Backport commits: " + + " ".join(s for s, cls, *_ in rows if cls != UNCLASSIFIABLE), + ] + ) + + +def warn_held(slug, repo, branch, annotate="warning", token=None): + """Take RC build / Promote on a stabilized branch: one annotation per held fix (Promote blocks on them unless + forced). Returns the summary line (the fixes, linked, and the RM's confirm command), or None when nothing is held. + """ + rows = held(slug, repo, branch, token=token) + other = slug if token else None # a token means another repo: name it + for s, cls, pr, _ in rows: + print(held_line(s, cls, pr, branch, annotate, other)) + return held_note(rows, branch, other) + + +def held_note(rows, branch, slug=None): + """One summary line for held() rows of stabilized `branch`, or None. slug: the other repo the rows are from.""" + if not rows: + return None + commits = ",".join( + pr if re.fullmatch(r"#\d+", pr) else s[:12] for s, _, pr, _ in rows + ) + link = ( + ( + lambda pr: f"[{slug}{pr}](https://github.com/{slug}/pull/{pr[1:]})" + if pr.startswith("#") + else pr + ) + if slug + else link_pr + ) + return ( + f"⚠ Held for the RM on {f'{slug} ' if slug else ''}`{branch}` (stabilized; Promote blocks unless forced): " + + ", ".join(f"{link(pr)} {cls}" for _, cls, pr, _ in rows) + + f". Pick with `gh workflow run backport.yaml{f' -R {slug}' if slug else ''} -f commits={commits} " + f"-f branches={branch} -f confirm_stabilized=true`" + ) + + +def rescan_targets(targets, listed): + """({branch: accept} to rescan, listed branches that are no target): fix targets owe plain fixes too (OWED), + the other security/critical targets (stabilized) only HELD. listed narrows, never adds. + """ + scan = {b: OWED for b in targets["fix"]} + for t in targets["security_critical"]: + scan.setdefault(t["branch"], HELD) + kept, others = only_listed(list(scan), listed) + return {b: scan[b] for b in kept}, others + + +def rescan_plan(targets, held_by_branch, confirm): + """Pure: (plan, awaiting, unclassifiable) from each rescanned branch's held() rows (main order, picked and + declined commits already dropped). plan: [(branch, sha, class, PR ref, title)], each branch in main order; + awaiting: [(sha, branch, class, PR ref)] on a stabilized branch without confirm; unclassifiable: + [(sha, branch, PR ref, title)], never picked. + """ + stabilized = { + t["branch"] for t in targets["security_critical"] if t["needs_rm_confirmation"] + } + plan, awaiting, unclassifiable = [], [], [] + for b, rows in held_by_branch.items(): + for sha, cls, pr, title in rows: + if cls == UNCLASSIFIABLE: + unclassifiable.append((sha, b, pr, title)) + elif b in stabilized and not confirm: + awaiting.append((sha, b, cls, pr)) + else: + plan.append((b, sha, cls, pr, title)) + return plan, awaiting, unclassifiable + + +def pick_row(slug, sha, b, cls, pr, title): + """Pick sha onto b and return its summary row; a failure never stalls the other branches.""" + try: + rel = rm.relevance( + REPO, sha, f"origin/{b}" + ) # advisory only: never skips the pick + except Exception as e: + rel = f"unknown ({type(e).__name__}: {(str(e).splitlines() or [''])[0]})".replace( + "|", r"\|" + ) # a table cell + print(f"::warning::{sha[:7]} relevance on {b}: {rel}") + try: + outcome, detail = pick(slug, sha, b, title, pr) + except ( + Exception + ) as e: # branches are independent: one failure never stalls the others + traceback.print_exc() + run("git", "cherry-pick", "--abort", check=False) + outcome, detail = "failed", f"{type(e).__name__}: {e}" + print(f"{sha[:7]} {cls} {b}: {outcome} {detail}") + return ( + sha, + pr, + cls, + b, + f"{outcome}{':' if outcome in ('skipped', 'failed') else ''} {detail}", + rel, + ) + + +def rescan(slug, targets, confirm, listed): + """Push mode: (rows, awaiting, {branch: missing}) over every target since its cut (see the module doc).""" + scan, others = rescan_targets(targets, listed) + rows = [] + for b in others: + print(f"::warning::{b} is not a target, skipped") + rows.append(("-" * 7, "-", "-", b, "skipped: not a target", "-")) + cuts = [rm.git(REPO, "merge-base", "origin/main", f"origin/{b}") for b in scan] + oldest = min( + (rm.git(REPO, "log", "-1", "--format=%cI", c) for c in cuts), default=None + ) + since = ( + (datetime.datetime.fromisoformat(oldest) - datetime.timedelta(days=1)).date() + if oldest + else None + ) + classes = ( + pr_classes(slug, since.isoformat()) if since else {} + ) # a day's margin for clock and merge-time skew + listed_n = len(classes) + held_by_branch = { + b: held(slug, REPO, b, accept=accept, cache=classes) for b, accept in scan.items() + } + print(f"rescan: {listed_n} PRs listed, {len(classes) - listed_n} fallback lookup(s)") + plan, awaiting, unclassifiable = rescan_plan(targets, held_by_branch, confirm) + for ( + sha, + b, + pr, + title, + ) in ( + unclassifiable + ): # a human dispatches with a class or declines it; re-warned each push + print( + f"::warning::{sha[:7]} ({pr}, {title}) cannot be classed for {b}: not picked" + ) + rows.append((sha, pr, UNCLASSIFIABLE, b, "skipped: unclassifiable", "-")) + for b, sha, cls, pr, title in plan: + rows.append(pick_row(slug, sha, b, cls, pr, title)) + missing = {} + for b in scan: + results = [r[4] for r in rows if r[3] == b] + picked = sum(res.startswith("merged") for res in results) + pending = sum( + res.startswith("pr ") or "pending" in res for res in results + ) # conflict PR, or merge held + missing[b] = len(held_by_branch[b]) - picked + print( + f"rescan {b}: {len(held_by_branch[b])} missing, {picked} picked, {pending} pending" + ) + return rows, awaiting, missing + + +PUSH_WAITS = (5, 20) # seconds before the 2nd and 3rd push attempt +PICK_TIMEOUT = ( + 30 * 60 +) # seconds: the pick job's timeout-minutes in backport.yaml; no run holds a head longer + + +def run_url(): + """This Actions run's URL, or None outside Actions.""" + e = os.environ + return ( + f"{e['GITHUB_SERVER_URL']}/{e['GITHUB_REPOSITORY']}/actions/runs/{e['GITHUB_RUN_ID']}" + if e.get("GITHUB_RUN_ID") + else None + ) + + +def push_head(slug, sha, branch, head, src, sleep=time.sleep): + """Push HEAD to head, retrying a rejection. None on success; after the last attempt ("failed", detail), + nothing pushed, the sentence on the run's ::error:: and as a comment on the source PR (Actions only). A pick + another run landed meanwhile is ("skipped", detail), with no comment. + """ + first = None + for wait in (*PUSH_WAITS, None): + r = run("git", "push", "-q", "origin", f"HEAD:refs/heads/{head}", check=False) + if r.returncode == 0: + return None + lines = (r.stderr.strip() or r.stdout.strip()).splitlines() or [ + f"exit {r.returncode}" + ] + # git prints "To " first; the "! [remote rejected] ..." or "error:" line names the cause + first = first or next( + ( + l.strip() + for l in lines + if l.lstrip().startswith(("!", "error:", "fatal:")) + ), + lines[0], + ) + if wait is not None: + sleep(wait) + run( + "git", + "fetch", + "-q", + "origin", + f"+refs/heads/{branch}:refs/remotes/origin/{branch}", + check=False, + ) + if rm.is_converged( + REPO, sha, f"origin/{branch}", main="origin/main" + ): # the run that won the head landed it + return ( + "skipped", + f"already on branch (push of {head} rejected: another run landed the pick)", + ) + url = run_url() + # convgate.missing_message matches this first line on the source PR: keep the two in step. + sentence = ( + f"Backport to {branch} failed: push of {head} rejected (run {url or 'outside Actions'}). " + f"Rerun Backport for {sha[:7]} on {branch}." + ) + print(f"::error::{sentence}") + if url and re.fullmatch(r"#\d+", src): + soft( + f"comment on {src}", + "gh", + "pr", + "comment", + src[1:], + "-R", + slug, + "--body", + f"{sentence}\n\n`{first}`", + ) + return "failed", f"push of {head} rejected {len(PUSH_WAITS) + 1} times: {first}" + + +CONFLICTS_RE = re.compile( + r"^Conflicts:\n\t", re.M +) # the block pick() adds to a conflict pick's message + + +def fix_labels(slug, src): + """The fix/* labels of the original PR src (`#n`), for its pick PR: Lint PR classes a pick PR by them too.""" + if not re.fullmatch(r"#\d+", src): + return [] + names = gh("api", f"repos/{slug}/pulls/{src[1:]}", "--jq", ".labels[].name").split( + "\n" + ) + return [x for x in names if x.startswith("fix/")] + + +def finish_open(slug, branch, head): + """The pick head already exists. An open unlabelled PR with only the pick on it is a clean pick whose merge + failed in an earlier run: squash it now, unless it is a conflict pick that lost its label. A conflict PR or a + person's commits wait for a person; a head with no open PR waits PICK_TIMEOUT for the run that pushed it, then + fails until a person deletes it or reopens its PR. + """ + pending = "skipped", f"conflict PR pending on `{head}`" + prs = json.loads( + gh( + "pr", + "list", + "-R", + slug, + "--head", + head, + "--base", + branch, + "--state", + "open", + "--json", + "number,headRefOid,labels", + ) + ) + if ( + not prs + ): # a running run opens its PR after the push and deletes the head after the squash + run( + "git", + "fetch", + "-q", + "origin", + f"+refs/heads/{head}:refs/remotes/origin/{head}", + ) + age = time.time() - int( + rm.git(REPO, "log", "-1", "--format=%ct", f"origin/{head}") + ) + if age < PICK_TIMEOUT: + return ( + "skipped", + f"head {head} is {int(age // 60)} min old with no open PR: another run is picking it", + ) + return ( + "failed", + f"head {head} exists with no open PR: delete it or reopen its PR, then rerun", + ) + if len(prs) != 1 or any(x["name"].startswith("backport-") for x in prs[0]["labels"]): + return pending + pr, remote = prs[0], f"origin/{branch}" + run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") + if rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}") != "1": + return pending + if rm.conflict_markers(REPO, pr["headRefOid"]) or CONFLICTS_RE.search( + rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) + ): + return "failed", ( + f"#{pr['number']} is a conflict pick without the backport-conflict label: label it " + "backport-conflict, then resolve and merge it, or close it to decline" + ) + try: + if not bm.squash( + slug, pr, rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) + ): + return "failed", f"#{pr['number']} merge pending check" + except RuntimeError as e: + return "failed", f"#{pr['number']} merge retry: {str(e).splitlines()[0]}" + soft( + f"delete of {head}", + "gh", + "api", + "-X", + "DELETE", + f"repos/{slug}/git/refs/heads/{head}", + ) + run("git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}") + return ( + "merged", + f"{rm.git(REPO, 'rev-parse', '--short=7', remote)} (#{pr['number']}, retried)", + ) + + +def pick(slug, sha, branch, title, src): + """Returns (outcome, detail); outcome in merged | pr | skipped | failed. title: PR title; src: original PR ref.""" + remote = f"origin/{branch}" + if rm.is_converged(REPO, sha, remote, main="origin/main"): + return "skipped", "already on branch" + head = f"backport/{branch.split('/', 1)[1]}/{sha[:7]}" + if ( + run( + "git", "ls-remote", "--exit-code", "--heads", "origin", head, check=False + ).returncode + == 0 + ): + return finish_open(slug, branch, head) + labels = [x for name in fix_labels(slug, src) for x in ("--label", name)] + run("git", "checkout", "-q", "--detach", remote) + r = run("git", "cherry-pick", "-x", sha, check=False) + if r.returncode == 0: + pick_sha = rm.git(REPO, "rev-parse", "HEAD") + if failed := push_head(slug, sha, branch, head, src): + return failed + url = gh( + "pr", + "create", + "-R", + slug, + "--base", + branch, + "--head", + head, + "--title", + title, + *labels, + "--body", + pr_body(sha, src, branch, head), + ) + n = int(url.rsplit("/", 1)[1]) + # backport_merge.squash: REST squash pinned to the head, with the pick's message. + # A required check GitHub still waits for -> False: PR and head stay, the next run reports it pending. + if not bm.squash( + slug, + {"number": n, "headRefOid": pick_sha}, + rm.git(REPO, "log", "-1", "--format=%B", pick_sha), + ): + return "failed", f"{url} opened, merge pending check" + soft( + f"delete of {head}", + "gh", + "api", + "-X", + "DELETE", + f"repos/{slug}/git/refs/heads/{head}", + ) + run( + "git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}" + ) # later picks stack on it + return "merged", f"{rm.git(REPO, 'rev-parse', '--short=7', remote)} (#{n})" + files = rm.git(REPO, "diff", "--name-only", "--diff-filter=U").split() + if not files: # not a conflict a human can resolve on a PR + empty = rm.git( + REPO, "rev-parse", "-q", "--verify", "CHERRY_PICK_HEAD", check=False + ).returncode == 0 and not rm.git( + REPO, "status", "--porcelain", "--untracked-files=no" + ) + run("git", "cherry-pick", "--abort", check=False) + if empty: + return "failed", ( + f"pick of {sha[:7]} onto {branch} is empty: the change is already there in another " + f"form. Record a decline: an empty commit on {branch} with the lines `(cherry picked " + f"from commit {sha})` and `Declined: already on {branch}`" + ) + return "failed", ( + (r.stderr.strip() or r.stdout.strip()).splitlines() + or [f"cherry-pick exit {r.returncode}"] + )[0] + # The message cherry-pick -x prepared (subject, body, trailer), minus git's commented hints. + with open(os.path.join(REPO, ".git", "MERGE_MSG")) as f: + msg = "".join(line for line in f if not line.startswith("#")).strip() + msg += "\n\nConflicts:\n" + "".join(f"\t{p}\n" for p in files) + run("git", "add", "-A") + run("git", "commit", "-q", "--no-verify", "-F", "-", input=msg) + if failed := push_head(slug, sha, branch, head, src): + return failed + url = gh( + "pr", + "create", + "-R", + slug, + "--base", + branch, + "--head", + head, + "--label", + "backport-conflict", + *labels, + "--title", + title, + "--body", + pr_body(sha, src, branch, head, files), + ) + return "pr", url + + +def push_title(slug, sha): + """The push summary's head line: the pushed commit and its class.""" + try: + cls, pr, _ = fix_class(slug, sha) + except ValueError as e: # a title Lint PR should have stopped: the picks are done, only this line is left + print(f"::warning::{sha[:7]}: fix class: {e}") + cls, pr = UNCLASSIFIABLE, "-" + return f"Backport · push {rm.commit_link(sha)} ({link_pr(pr)}, {cls or 'not a fix'})" + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--state", required=True) + mode = p.add_mutually_exclusive_group(required=True) + mode.add_argument( + "--commits", + help="main SHAs or merged PR numbers, comma- or space-separated, any order", + ) + mode.add_argument( + "--rescan", action="store_true", help="every target since its cut (push to main)" + ) + p.add_argument("--confirm-stabilized", action="store_true") + p.add_argument( + "--branches", + default="", + help="comma list: only these release branches (default every target)", + ) + a = p.parse_args() + slug = os.environ["GITHUB_REPOSITORY"] + with open(a.state) as f: + state = json.load(f) + targets = state["backport_targets"] + listed = [b.strip() for b in a.branches.split(",") if b.strip()] + if a.rescan: + rows, awaiting, missing = rescan(slug, targets, a.confirm_stabilized, listed) + else: + rows, awaiting = dispatch(slug, targets, a.commits, a.confirm_stabilized, listed) + failed = sum(res.startswith("failed") for *_, res, _ in rows) + if awaiting: + print("\n".join(held_warnings(awaiting))) # a held pick is never silent + + now = datetime.datetime.now(datetime.UTC) + days = lambda iso: ( + now - datetime.datetime.fromisoformat(iso.replace("Z", "+00:00")) + ).days + conflicts = sorted( + (p["number"], p["baseRefName"], days(p["createdAt"])) for p in conflict_prs(slug) + ) + held_rows = [ + (s, b, c, pr, days(rm.git(REPO, "log", "-1", "--format=%cI", s))) + for s, b, c, pr in awaiting + ] + pending = pending_block(conflicts, held_rows) + if a.rescan: + head = push_title( + slug, os.environ.get("GITHUB_SHA") or rm.git(REPO, "rev-parse", "origin/main") + ) + states = {r["branch"]: r["state"] for r in state["branches"]} + table = [] + for b in missing: + tag, unbuilt = latest_tag(REPO, b) + table.append( + status_row( + b, + states.get(b, "-"), + tag, + unbuilt, + missing[b], + [n for n, base, _ in conflicts if base == b], + sum(x[1] == b for x in awaiting), + ) + ) + out = push_summary(head, table, this_run(rows), pending) + else: # the person named these commits: every row answers them, skipped ones included + out = [ + "### Backport · dispatch", + "", + "| commit | PR | class | branch | result | relevance |", + "|---|---|---|---|---|---|", + ] + out += [ + f"| `{s[:7]}` | {pr} | {c} | {b} | {res} | {rel} |" + for s, pr, c, b, res, rel in rows + ] + out += [""] + pending + print("\n".join(out)) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write("\n".join(out) + "\n") + return 1 if failed else 0 + + +def dispatch(slug, targets, commits, confirm, listed): + """Dispatch mode: (rows, awaiting) for the given commits only.""" + items = [ + c for c in re.split(r"[,\s]+", commits) if c + ] # commas or spaces: the owed-picks line pastes as is + shas = [resolve(slug, c) for c in items] + if not items or None in shas: + bad = [c for c, s in zip(items, shas, strict=False) if s is None] + sys.exit(f"::error::refused: not merged commits on main: {bad or 'none given'}") + shas = main_order(REPO, shas) + + rows, awaiting = [], [] + for sha in shas: + try: + cls, pr, title = fix_class(slug, sha) + except ValueError as e: # Lint PR should have stopped this PR + print(f"::error::{sha[:7]}: fix class: {e}") + rows.append((sha, "?", "error", "-", f"skipped: {e}", "-")) + continue + if cls is None: + route = ( + f"; label {pr} fix/needs-backport (or the fix label it needs) and rerun" + if pr.startswith("#") + else "" + ) + rows.append((sha, pr, "none", "-", f"skipped: not a fix{route}", "-")) + continue + if cls == "fix": + branches = targets["fix"] + else: + branches = [ + t["branch"] + for t in targets["security_critical"] + if confirm or not t["needs_rm_confirmation"] + ] + awaiting += [ + (sha, t["branch"], cls, pr) + for t in targets["security_critical"] + if t["needs_rm_confirmation"] + and not confirm + and (not listed or t["branch"] in listed) + ] + branches, others = only_listed(branches, listed) + others = [b for b in others if (sha, b, cls, pr) not in awaiting] + for b in others: + print(f"::warning::{sha[:7]} ({cls}, PR {pr}): {b} is not a target, skipped") + rows.append((sha, pr, cls, b, "skipped: not a target", "-")) + if not branches and not others: + rows.append((sha, pr, cls, "-", "skipped: no target branch", "-")) + rows += [pick_row(slug, sha, b, cls, pr, title) for b in branches] + return rows, awaiting + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py new file mode 100644 index 0000000000..628bfaaa6c --- /dev/null +++ b/.github/release/backport_merge.py @@ -0,0 +1,359 @@ +"""The bot's side of a backport PR: Backport's squash of a clean pick, and the reaction when a person closes or +reopens one. + +Usage: backport_merge.py --pr N --sender LOGIN (Backport PR workflow, on `pull_request_target` closed or reopened +for a backport/* head into release/*; LOGIN is the event's sender; a failed run is recovered with "Re-run jobs"). +Anything but a same-repo backport/X.Y/ head into release/X.Y, opened by the bot, is refused. + +People land a conflict PR with the normal buttons (resolve, one approving review, Squash and merge) or close it. The PR's +state is read again here, so a re-run or a repeated event acts on what the PR is now. Labels on backport PRs are +the bot's: each case sets exactly one `backport-*` label and removes the others. +- merged: label `backport-merged`; delete the head if it is still there. +- closed unmerged: a decline. The reason is the closer's newest comment made in the 2 minutes up to the close (the + close comment), first line, a leading `Declined:` dropped; with none, `Closed by @ without a reason (PR + #)`. Push an empty commit onto the base with the pick's message, the trailer and a final `Declined: ` + line; label `backport-declined`; comment `Recorded as a decline on (). To undo: ...` once per close. + The head is kept so the PR can be reopened. A pick or decline already on the base (newest first) -> no commit, + label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never + names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the + head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. +- open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and + the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. +Run from a full clone of main, GH_TOKEN = bot token. +""" + +import argparse +import datetime +import json +import os +import subprocess +import sys +import time + +import relmgr as rm + +MERGED, DECLINED, CONFLICT = "backport-merged", "backport-declined", "backport-conflict" +REASON_WINDOW = datetime.timedelta(minutes=2) + + +def is_backport(head, base): + """Is head a pick head for base (backport/X.Y/ into release/X.Y)?""" + m = rm.BACKPORT_HEAD_RE.fullmatch(head) + return bool(m) and base == f"release/{m[1]}" + + +def gh(*args, token=None): + env = {**os.environ, "GH_TOKEN": token} if token else None + r = subprocess.run(["gh", *args], capture_output=True, text=True, env=env) + if r.returncode: + raise RuntimeError( + f"gh {' '.join(args[:2])} (exit {r.returncode}): {r.stderr.strip()}" + ) + return r.stdout.strip() + + +def decline_reason(comments, sender, closed_at): + """First line of the closer's newest comment in the REASON_WINDOW up to closed_at, a leading `Declined:` dropped; + None if there is none or it is empty. + """ + at = lambda t: datetime.datetime.fromisoformat(t.replace("Z", "+00:00")) + end = at(closed_at) + for c in sorted(comments, key=lambda c: c["created_at"], reverse=True): + if c["user"] == sender and end - REASON_WINDOW <= at(c["created_at"]) <= end: + line = (c["body"].strip().splitlines() or [""])[0] + return line.removeprefix("Declined:").strip() or None + return None + + +def head_messages(pr): + """Messages of the PR head's commits above the base, oldest first. Fetched through refs/pull/N/head, which + outlives the deleted head branch. + """ + n, base, sha = pr["number"], pr["baseRefName"], pr["headRefOid"] + rm.git( + ".", + "fetch", + "-q", + "origin", + f"+refs/heads/{base}:refs/remotes/origin/{base}", + f"+refs/pull/{n}/head:refs/backport/{n}", + ) + return [ + rm.git(".", "log", "-1", "--format=%B", c) + for c in rm.git(".", "rev-list", "--reverse", f"origin/{base}..{sha}").split() + ] + + +def first_pick(pr): + """(message, [original]) for the decline: the main commit the head's first commit picks, its message plus the + pick trailer. ("", []) when that commit has no single pick trailer naming the head's sha7 on main. + """ + msgs = head_messages(pr) + o = rm.pick_trailer(pr["headRefName"], msgs[0] if msgs else "") + if not o or not rm.on_main(".", o, "origin/main"): + return "", [] + full = rm.git(".", "rev-parse", f"{o}^{{commit}}") + return ( + f"{rm.git('.', 'log', '-1', '--format=%B', full).strip()}\n\n(cherry picked from commit {full})", + [full], + ) + + +def landed(originals, commits): + """First (sha, message) in commits whose pick trailer names one of originals; None if the pick is not there.""" + for sha, msg in commits: + if set(rm.PICK_RE.findall(msg)) & set(originals): + return sha + return None + + +def base_picks(base, originals): + """(sha, message) of commits on origin/ whose message names one of originals in a pick trailer.""" + greps = [f"--grep=(cherry picked from commit {o}" for o in originals] + out = rm.git(".", "log", "--format=%H%x00%B%x01", "-F", *greps, f"origin/{base}") + return [tuple(c.strip("\n").split("\0", 1)) for c in out.split("\x01") if c.strip()] + + +def base_tip(base): + """(commit, tree) of origin/: the parent and the tree of an empty decline commit.""" + return rm.git(".", "rev-parse", f"origin/{base}"), rm.git( + ".", "rev-parse", f"origin/{base}^{{tree}}" + ) + + +SQUASH_WAITS = (5, 20) # seconds before the 2nd and 3rd merge attempt + + +def squash(slug, pr, msg, sleep=time.sleep): + """REST squash merge of the PR head with msg as the commit message; False while a required check is pending. + + GitHub can answer a merge of a just-opened PR with 405 "Base branch was modified" although nothing moved the + base; that answer is retried (3 attempts, 5 s then 20 s apart), then raised. + """ + title, _, body = msg.partition("\n") + for wait in (*SQUASH_WAITS, None): + try: + gh( + "api", + "-X", + "PUT", + f"repos/{slug}/pulls/{pr['number']}/merge", + "-f", + "merge_method=squash", + "-f", + f"sha={pr['headRefOid']}", + "-f", + f"commit_title={title}", + "-f", + f"commit_message={body.strip()}", + ) + return True + except RuntimeError as e: + if ( + "status check" in str(e).lower() + ): # a required check GitHub still waits for: the caller reports it + return False + if wait is None or "base branch was modified" not in str(e).lower(): + raise + print(f"#{pr['number']}: {e}; merge retried in {wait} s") + sleep(wait) + + +def set_label(slug, pr, label): + """Make `label` the PR's only backport-* label (REST: works on closed PRs).""" + have = {x["name"] for x in pr["labels"]} + for old in sorted(x for x in have if x.startswith("backport-") and x != label): + gh("api", "-X", "DELETE", f"repos/{slug}/issues/{pr['number']}/labels/{old}") + if label not in have: + gh( + "api", + "-X", + "POST", + f"repos/{slug}/issues/{pr['number']}/labels", + "-f", + f"labels[]={label}", + ) + + +def delete_head(slug, head): + """Delete the head branch; already gone (autodelete on merge, a person, a re-run) is fine.""" + try: + gh("api", "-X", "DELETE", f"repos/{slug}/git/refs/heads/{head}") + except RuntimeError as e: + if ( + "404" not in str(e) + and "422" not in str(e) + and "Reference does not exist" not in str(e) + ): + raise + + +def comments(slug, n): + out = gh( + "api", + "--paginate", + f"repos/{slug}/issues/{n}/comments", + "--jq", + ".[] | {created_at, body, user: .user.login}", + ) + return [json.loads(line) for line in out.splitlines()] + + +def decline(slug, pr, reason): + """(sha, written): the pick or decline for this PR's originals on the base, newest first, else a new decline.""" + n, base = pr["number"], pr["baseRefName"] + msg, originals = first_pick(pr) + if not originals: + gh( + "pr", + "comment", + str(n), + "-R", + slug, + "--body", + "Bot could not record the decline: the first commit's message needs exactly one `(cherry picked from " + "commit
)` line, naming the main commit of this branch's name. Decline it with a hand-made empty " + "commit carrying that line, or ask the release manager.", + ) + raise RuntimeError( + f"#{n}: no single pick trailer for {pr['headRefName']}, decline not recorded" + ) + if done := landed( + originals, base_picks(base, originals) + ): # re-run, or a close after a reopen: nothing to write + return done, False + # Empty commit = the base's own tree. REST, so author and committer are the bot token's user, as on merge. + tip, tree = base_tip(base) + new = json.loads( + gh( + "api", + "-X", + "POST", + f"repos/{slug}/git/commits", + "-f", + f"message={msg}\n\nDeclined: {reason}", + "-f", + f"tree={tree}", + "-f", + f"parents[]={tip}", + ) + )["sha"] + gh( + "api", + "-X", + "PATCH", + f"repos/{slug}/git/refs/heads/{base}", + "-f", + f"sha={new}", + "-F", + "force=false", + ) + return new, True + + +def close(slug, n, sender): + return record(slug, n, sender)["result"] + + +def record(slug, n, sender): + """What close did: {result, base, sha, reason}; sha is the squash, decline or existing pick, reason the decline's.""" + pr = json.loads( + gh( + "pr", + "view", + str(n), + "-R", + slug, + "--json", + "number,state,closedAt,baseRefName,headRefName,headRefOid,labels,isCrossRepository,mergeCommit,author", + ) + ) + base, head = pr["baseRefName"], pr["headRefName"] + if not is_backport(head, base) or pr["isCrossRepository"]: + raise RuntimeError( + f"#{n} is not a backport PR ({head} into {base}): nothing recorded" + ) + # A head name and trailer are strings any writer can push; only a PR the bot opened is its pick. GH_TOKEN is the + # bot token Backport opens pick PRs with, so its own login is the one to compare. + if (author := pr["author"]["login"]) != (bot := gh("api", "user", "--jq", ".login")): + raise RuntimeError( + f"#{n} was opened by @{author}, not by the bot (@{bot}): nothing recorded" + ) + if pr["state"] == "OPEN": + set_label(slug, pr, CONFLICT) + return {"result": "reopened", "base": base, "sha": None, "reason": None} + if pr["state"] == "MERGED": + set_label(slug, pr, MERGED) + delete_head(slug, head) + return { + "result": "merged", + "base": base, + "sha": (pr.get("mergeCommit") or {}).get("oid"), + "reason": None, + } + cs = comments(slug, n) + reason = ( + decline_reason(cs, sender, pr["closedAt"]) + or f"Closed by @{sender} without a reason (PR #{n})" + ) + sha, written = decline(slug, pr, reason) + set_label(slug, pr, DECLINED) + if not any( + c["body"].startswith("Recorded as a decline") + and c["created_at"] >= pr["closedAt"] + for c in cs + ): + gh( + "pr", + "comment", + str(n), + "-R", + slug, + "--body", + f"Recorded as a decline on {base} ({sha[:7]}). To undo: reopen this PR and merge it.", + ) + result = ( + f"declined {sha[:7]}" if written else f"skipped: already on {base} as {sha[:7]}" + ) + return {"result": result, "base": base, "sha": sha, "reason": reason} + + +def summary(n, sender, rec): + """The step summary block: what happened to the PR, by whom, and how to undo it.""" + base, sha, pr = rec["base"], rec["sha"], rm.pr_link(n) + head = f"**Backport PR** {pr} on `{base}`: " + if rec["result"].startswith("declined"): + return [ + head + f"**declined** as {rm.commit_link(sha)}", + f'Reason: "{rec["reason"]}" (@{sender})', + f"Undo: reopen {pr} and merge it.", + ] + if rec["result"] == "merged": + squash = rm.commit_link(sha) if sha else "the squash" + return [ + head + f"**merged** as {squash} (@{sender})", + f"Undo: revert {squash} on `{base}`.", + ] + if rec["result"] == "reopened": + return [ + head + f"**reopened** (@{sender})", + "Next: merge it to land the pick, or close it again to keep the decline.", + ] + return [head + f"**skipped**: already on `{base}` as {rm.commit_link(sha)}"] + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--pr", type=int, required=True) + p.add_argument("--sender", required=True) + a = p.parse_args() + rec = record(os.environ["GITHUB_REPOSITORY"], a.pr, a.sender) + print(f"#{a.pr}: {rec['result']}") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(" \n".join(summary(a.pr, a.sender, rec)) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release/convgate.py b/.github/release/convgate.py new file mode 100644 index 0000000000..876a5472a9 --- /dev/null +++ b/.github/release/convgate.py @@ -0,0 +1,531 @@ +"""Convergence gate: scan every other supported branch of a target, both directions, so an older branch +never ships a fix a newer one lacks. + +Usage: convgate.py --branch release/X.Y --target T [--promote-ref vX.Y.Z-rc.N] [--state state.json] [--repo DIR] + [--oss-repo DIR] [--annotate warning|error] [--out FILE] + +Wraps relmgr.convergence_report unchanged and adds four things: +- scope: relstate `supported[T]` minus the branch. Retired and not-enabled branches are out by construction. +- direction: an older branch's picks are checked against the newer branch, whichever side is being promoted. +- reference: a newer branch with a promoted stable tag for T is scanned at that tag (what ships next), not at its tip, + even before it is GA (on-prem: promoted, chart not published yet). The branch being promoted is scanned at --promote-ref. +- declines shown: an older line's pick the newer line declined prints as DECLINED, with the reason and the decline + commit (relmgr.decline_records). It counts as converged. A decline on the older line owes the newer line nothing. +- release-branch-only window: an RBO-ABSENT row is kept only for a commit dated after the newer line's latest GA + promoted stable tag for T (its cut, the merge-base with main, when it has none yet) that touches a path outside .github/. + The RM saw older ones at the last promote, and workflow-only commits carry nothing a product line needs. A dropped + commit leaves the counts and the report. +- OSS pins (a repo whose lines pin an OSS tag, relstate PINS): the same scan in a clone of the OSS repo, the older + line's pinned OSS tag into the newer line's, so an upgrade never loses an OSS fix. Each line is read at the same + reference as above. A MISSING row names where to act: the OSS line (OSS Backport) or a new RC of the newer line. +Exit 2 iff any MISSING or an older line is not GA yet (promote order), 0 otherwise, 1 on error (never a silent +pass). Take RC build warns on 2; Promote blocks. +Each MISSING row names what blocks the promote and where to act (the open Backport conflict PR, if any, or +the Backport run whose push was rejected). +checks() turns a gate result into the checks table both summaries lead with, and the full rows folded below it. +""" + +import argparse +import datetime as dt +import json +import os +import re +import subprocess +import sys +import traceback + +import oss +import relmgr as rm +import relstate as rs + +STATUSES = ("CONVERGED", "DECLINED", "MISSING", "UNCLASSIFIABLE", "RBO-ABSENT") + + +def conflict_pr(head): + """(number, url) of the open Backport conflict PR on `head`, or None.""" + r = subprocess.run( + [ + "gh", + "pr", + "list", + "-R", + os.environ["GITHUB_REPOSITORY"], + "--head", + head, + "--state", + "open", + "--json", + "number,url", + ], + capture_output=True, + text=True, + ) + if r.returncode: # never report "no PR found" when the lookup itself failed + raise RuntimeError(f"gh pr list --head {head}: {r.stderr.strip()}") + prs = json.loads(r.stdout or "[]") + return (prs[0]["number"], prs[0]["url"]) if prs else None + + +def push_failure(number, branch): + """Run URL in the newest `Backport to failed:` comment Backport left on PR #number, or None.""" + r = subprocess.run( + [ + "gh", + "api", + "--paginate", + f"repos/{os.environ['GITHUB_REPOSITORY']}/issues/{number}/comments", + "--jq", + f'.[] | .body | split("\\n")[0] | select(startswith("Backport to {branch} failed:"))', + ], + capture_output=True, + text=True, + ) + if r.returncode: # never fall back to "no PR found" when the lookup itself failed + raise RuntimeError(f"gh api issues/{number}/comments: {r.stderr.strip()}") + m = re.search(r"\(run (\S+)\)", (r.stdout.strip().splitlines() or [""])[-1]) + return m and m[1] + + +def blocked(branch, target, annotate): + """The lead of every MISSING message: an error where it blocks (Promote), a warning where it will (Take RC).""" + return ( + f"Cannot promote {branch} for {target}" + if annotate == "error" + else f"Promote of {branch} for {target} will be blocked" + ) + + +def missing_message( + repo, branch, target, older, newer, orig, pr, annotate, unshipped=None, failure=None +): + """What blocks the promote of `branch` and where to act: the fix `orig` is on `older` but not on `newer`. + unshipped: the tag `newer` is scanned at (its latest stable, or the RC being promoted when `newer` is + `branch`) when the fix is on `newer`'s tip but not in that tag. + failure(pr number, newer): the run URL of Backport's push-failure comment on the source PR, or None. + """ + o7 = orig[:7] + subject = rm.git( + repo, "log", "-1", "--format=%s", orig, check=False + ).stdout.strip() # orig may not be on main + m = re.search(r"\(#(\d+)\)$", subject) + fix = f"fix {o7}" + (f" (PR #{m[1]})" if m else "") + head = blocked(branch, target, annotate) + if ( + unshipped and newer == branch and rs.VER_RE.fullmatch(unshipped)[4] + ): # an RC of this line, cut before the fix + return f"{head}: {fix} is on {newer} but not in RC {unshipped}: take a new RC build first." + if unshipped: # nothing to pick or decline: the fix only waits for newer's next patch + return f"{head}: {fix} is on {newer} but not in its latest stable {unshipped}: promote a {newer} patch first." + if branch == older: + msg = f"{head}: {fix} is not on newer {newer} yet." + act = ( + f"Conflict PR #{pr[0]} on {newer}: resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." + if pr + else None + ) + else: + msg = f"{head}: {fix} is on older {older} but not on this line." + act = ( + f"Conflict PR #{pr[0]} is on this branch: resolve, get one approval and merge, or close the PR to " + "decline (the close comment is the reason)." + if pr + else None + ) + if ( + not act and m and failure and (url := failure(int(m[1]), newer)) + ): # backport.push_head wrote it + act = f"Backport for {o7} to {newer} failed: push rejected (run {url}). Rerun Backport for {o7} on {newer}." + return f"{msg} {act or f'No Backport PR found for it: run Backport for {o7}, or decline it on {newer}.'}" + + +def oss_pins(repo, oss_repo, branch, target, scope, ref, rows, annotate): + """(missing, report lines, per-line results): each older line's pinned OSS tag scanned into the newer line's.""" + main, oss_refs = "refs/remotes/origin/main", rs._branches(oss_repo) + pins = {n: oss.pin(repo, ref(n)) for n in [branch, *scope]} + for p in set(pins.values()): + if not rs._tag_commit(oss_repo, p): + raise RuntimeError(f"no OSS tag {p} in {oss_repo}") + head = blocked(branch, target, annotate) + missing, lines, out = 0, [], [f"### OSS pins: `{branch}` pins `{pins[branch]}`", ""] + for s in scope: + older, newer = sorted((s, branch), key=lambda n: rs._key(n.split("/", 1)[1])) + p_old, p_new, line_new = pins[older], pins[newer], oss.line_of(pins[newer]) + found = ( + [] + if p_old == p_new + else [ + r + for r in rm.convergence_report( + oss_repo, + f"refs/tags/{p_old}", + f"refs/tags/{p_new}", + main=main, + newer_branch=oss_refs.get(line_new), + ) + if r[1] in ("CONVERGED", "MISSING") + ] + ) + n_miss = sum(st == "MISSING" for _, st in found) + missing += n_miss + summary = f"{len(found)} OSS picks, {len(found) - n_miss} in {p_new}" + ( + f", {n_miss} missing" if n_miss else "" + ) + line = { + "line": s, + "side": "older" if s == older else "newer", + "picks": len(found), + "summary": summary, + "todo": [], + "rbo": [], + "state": rows[s]["targets"].get(target, {}).get("state"), + "oss": True, + } + lines.append(line) + out.append( + f"**{s}** ({line['side']}; `{older}` pins `{p_old}`, `{newer}` pins `{p_new}`): {summary}" + ) + for c, st in found: + if st != "MISSING": + continue + for o in rm.picked_from(oss_repo, c): + m = re.search( + r"\(#(\d+)\)$", rm.git(oss_repo, "log", "-1", "--format=%s", o) + ) + fix = f"OSS fix {o[:7]}" + (f" (OSS PR #{m[1]})" if m else "") + on_line = line_new in oss_refs and rm.is_converged( + oss_repo, o, oss_refs[line_new], main + ) + act = ( + f"take a new RC of {newer} to pin OSS {line_new}'s tip" + if on_line + else f"pick it onto OSS {line_new} (OSS Backport), then take a new RC of {newer}" + ) + msg = f"{head}: {fix} is in {older}'s OSS pin {p_old} but not in {newer}'s pin {p_new}: {act}." + print(f"::{annotate}::{msg}") + out.append(f"- `{c[:12]}` MISSING original {o[:12]}") + out.append(f" - {msg}") + line["todo"].append(act) + out.append("") + return missing, out, lines + + +def gate( + repo, + branch, + target, + state, + promote_ref=None, + annotate="warning", + lookup=conflict_pr, + failure=push_failure, + oss_repo=None, +): + """(exit code, report lines, per-line results for checks()). oss_repo: a clone of rs.PINS, for the OSS pins.""" + refs = rs._branches(repo) + main = rs._main(repo) + rows = {b["branch"]: b for b in state["branches"]} + if branch not in refs or branch not in rows: + raise RuntimeError(f"{branch} not found") + + def tag_ref(tag): + if not rs._tag_commit(repo, tag): + raise RuntimeError(f"no git tag {tag}") + return f"refs/tags/{tag}" + + def ref(name): # a promoted branch ships its latest stable tag for T, not its tip + if name == branch and promote_ref: + return tag_ref(promote_ref) + p = rows[name]["targets"].get(target, {}).get("promoted") + return tag_ref(p["tag"]) if p else refs[name] + + scope = [s for s in state["supported"][target] if s != branch] + out = [f"### Convergence gate: `{branch}` for {target} at `{ref(branch)}`", ""] + if not scope: + out.append("scanned branches: none") + missing, lines = 0, [] + for s in scope: + older, newer = sorted((s, branch), key=lambda n: rs._key(n.split("/", 1)[1])) + promoted = rows[newer]["targets"].get(target, {}).get("promoted") + since = int( + rm.git( + repo, + "for-each-ref", + "--format=%(creatordate:unix)", + f"refs/tags/{promoted['tag']}", + ) + if promoted + else rm.git( + repo, + "log", + "-1", + "--format=%ct", + rm.git(repo, "merge-base", main, refs[newer]), + ) + ) + rows_out, n_picks = [], 0 + recs = rm.decline_records(repo, refs[newer], main) # declines on newer's tip + rec = lambda o, recs=recs: next( + (v for k, v in recs.items() if k.startswith(o) or o.startswith(k)), None + ) # trailers may be abbreviated + for c, status in rm.convergence_report( + repo, refs[older], ref(newer), main=main, newer_branch=refs[newer] + ): + if status == "RBO-ABSENT" and ( + int(rm.git(repo, "log", "-1", "--format=%ct", c)) <= since + or not any( + not f.startswith(".github/") + for f in rm.git(repo, "show", "--format=", "--name-only", c).split() + ) + ): + continue # outside the window, or nothing outside .github/ (an empty commit included) + origs = rm.picked_from(repo, c) + n_picks += bool(origs) + decline = ( + rec(origs[0]) + if status == "CONVERGED" and origs and all(rec(o) for o in origs) + else None + ) + if decline: + status = "DECLINED" + rows_out.append((c, status, origs, decline)) + n = {st: sum(r[1] == st for r in rows_out) for st in STATUSES} + missing += n["MISSING"] + state_s = rows[s]["targets"].get(target, {}).get("state") + shipped = ( + "the RC" + if not promoted or newer == branch and promote_ref + else promoted["tag"] + ) # what newer is scanned at + summary = f"{n_picks} picks, {n['CONVERGED']} in {shipped}" + "".join( + f", {n[k]} {word}" + for k, word in ( + ("DECLINED", f"declined on {newer}"), + ("MISSING", "missing"), + ("UNCLASSIFIABLE", "unclassifiable"), + ) + if n[k] + ) + if n["RBO-ABSENT"]: + summary += f" · {n['RBO-ABSENT']} release-branch-only to check" + line = { + "line": s, + "side": "older" if s == older else "newer", + "picks": n_picks, + "summary": summary, + "todo": [], + "rbo": [], + "state": state_s, + } + lines.append(line) + out.append( + f"**{s}** ({line['side']}, {state_s or 'not enabled'} for {target}; " + f"`{refs[older]}` into `{ref(newer)}`): {summary}" + ) + for c, status, origs, decline in rows_out: + if status == "DECLINED": + d, why = decline + out.append( + f"- `{c[:12]}` DECLINED on {newer} original {', '.join(o[:12] for o in origs)}: {why} ({d[:7]})" + ) + elif ( + status == "RBO-ABSENT" + ): # no original to name: the ticket and the commit instead + body = rm.git(repo, "log", "-1", "--format=%s%n%n%b", c) + tickets, title = ( + list(dict.fromkeys(rm.RBO_RE.findall(body))), + body.splitlines()[0], + ) + lead = ( + "" if title.split(" ", 1)[0] in tickets else ", ".join(tickets) + " " + ) # the title may name it + what = f"{lead}{title} ({rm.commit_link(c)})" + out.append(f"- `{c[:12]}` RBO-ABSENT {what}") + line["rbo"].append(f"on `{older}`: {what}, not on `{newer}`") + elif status != "CONVERGED": + out.append( + f"- `{c[:12]}` {status} original {', '.join(o[:12] for o in origs) or '-'}" + ) + if status == "MISSING": + for o in ( + origs + ): # the developer's next step, on the run page and in the summary + pr = lookup(f"backport/{newer.split('/', 1)[1]}/{o[:7]}") + shipped = ref( + newer + ) # newer's promoted stable tag when it has one, else its tip + unshipped = ( + shipped.removeprefix("refs/tags/") + if not pr + and shipped != refs[newer] + and rm.is_converged(repo, o, refs[newer], main) + else None + ) + msg = missing_message( + repo, + branch, + target, + older, + newer, + o, + pr, + annotate, + unshipped, + failure, + ) + print(f"::{annotate}::{msg}") + out.append(f" - {msg}") + line["todo"].append( + f"merge or close conflict PR {rm.pr_link(pr[0])}" + + ( + ", then Take RC again" + if newer == branch + else f" on `{newer}`" + ) + if pr + else msg.split(": ", 1)[1] + ) + out.append("") + if oss_repo: + n, o_out, o_lines = oss_pins( + repo, oss_repo, branch, target, scope, ref, rows, annotate + ) + missing, out, lines = missing + n, out + o_out, lines + o_lines + return (2 if missing else 0), out, lines + + +def next_steps(lines, order): + """The do-this items in checks-table order, each once and without a closing full stop (the Next line joins them): + every line's todo, then each promote-order blocker. + """ + return list( + dict.fromkeys( + [t.removesuffix(".") for x in lines for t in x["todo"]] + + [f"promote or retire `{o.split(' is not GA')[0]}`" for o in order] + ) + ) + + +def checks(lines, order, out, split=False): + """(checks table, folded gate detail) from gate()'s lines and out and relstate.promote_blockers' order. + split: one fixes row per direction (Promote) instead of one for every other line (the Take RC preview). + Blocking and to-check rows name what to do; healthy rows show counts only. A line is named only when it is + stabilizing for the target or has something to do or check; the rest are one trailing count. + """ + + def cell(parts, none="-"): + return "
".join(parts).replace("|", "\\|") or none + + def fixes(name, ls): + named = [x for x in ls if x["state"] == "stabilizing" or x["todo"] or x["rbo"]] + rest = [x for x in ls if x not in named] + counts = [f"`{x['line']}` {x['summary']}" for x in named] + if rest: + side = rest[0]["side"] if len({x["side"] for x in rest}) == 1 else "other" + counts.append( + f"{len(rest)} {side} line{'s' * (len(rest) > 1)} converged ({sum(x['picks'] for x in rest)} picks)" + ) + todo = next_steps(ls, []) + return ( + "⛔" if todo else "✅", + name, + cell(counts, "no other supported line"), + cell(todo), + ) + + own, pinned = ( + [x for x in lines if not x.get("oss")], + [x for x in lines if x.get("oss")], + ) + if split: + rows = [ + fixes(f"fixes match {side} lines", [x for x in own if x["side"] == side]) + for side in ("newer", "older") + ] + else: + rows = [fixes("fixes match the other lines", own)] + if pinned: + rows.append(fixes("OSS pins match the other lines", pinned)) + rows.append( + ( + "⛔" if order else "✅", + "promote order", + cell([o.split(":")[0] for o in order], "no older line is waiting to go GA"), + cell(next_steps([], order)), + ) + ) + rbo = [r for x in lines for r in x["rbo"]] + rows.append( + ( + "⚠" if rbo else "✅", + "release-branch-only to check", + cell(rbo, "none"), + "check whether the newer line needs its own equivalent" if rbo else "-", + ) + ) + table = ["| check | result | do this |", "|---|---|---|"] + [ + f"| {i} {c} | {r} | {d} |" for i, c, r, d in rows + ] + return table, ["
Gate detail", "", *out, "
"] + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--branch", required=True) + p.add_argument("--target", required=True) + p.add_argument( + "--promote-ref", + help="the RC being built or promoted; default: the branch's own reference", + ) + p.add_argument("--state", help="relstate JSON; default: derive it now") + p.add_argument("--repo", default=".") + p.add_argument( + "--oss-repo", + default=os.environ.get("OSS_REPO"), + help="clone of the repo this one pins (relstate PINS); required when PINS is set. Default: $OSS_REPO", + ) + p.add_argument( + "--annotate", + choices=("warning", "error"), + default="warning", + help="annotation level of MISSING rows: error where they block (Promote), warning where not", + ) + p.add_argument( + "--out", + default=os.environ.get("GITHUB_STEP_SUMMARY"), + help="file the checks table and gate detail are appended to; default: the step summary", + ) + a = p.parse_args(argv) + try: + if rs.PINS and not a.oss_repo: + raise RuntimeError( + f"--oss-repo (or OSS_REPO) is required: this repo pins {rs.PINS}" + ) + if a.state: + with open(a.state) as f: + state = json.load(f) + else: + state = rs.derive(a.repo, dt.datetime.now(dt.UTC)) + code, out, lines = gate( + a.repo, + a.branch, + a.target, + state, + a.promote_ref, + annotate=a.annotate, + oss_repo=a.oss_repo if rs.PINS else None, + ) + order = rs.promote_blockers(state, a.branch, a.target) + except Exception as e: # git, registry or input failure: an error, never a pass + traceback.print_exc() + print(f"::error::convgate: {type(e).__name__}: {e}") + return 1 + print("\n".join(out + order)) + if a.out: + table, detail = checks(lines, order, out) + with open(a.out, "a") as f: + f.write("\n".join(table + [""] + detail) + "\n\n") + return 2 if code or order else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release/oss.py b/.github/release/oss.py new file mode 100644 index 0000000000..349ea30663 --- /dev/null +++ b/.github/release/oss.py @@ -0,0 +1,361 @@ +"""OSS release lines, driven by git from the cloud release workflows. Cloud release/X.Y pins OSS release/X.Y (same +minor): `unstract-platform-version` in pyproject.toml names one OSS tag, ee-vX.Y.Z-rc.N or ee-vX.Y.Z. + +Usage (OSS_REPO: a clone of the OSS repo with full history, tags and origin/release/* heads, that can push): + oss.py cut --line X.Y [--slack-url URL] cut OSS release/X.Y from OSS main; prints ee-vX.Y.0-rc.1 + oss.py rc --branch release/X.Y [--slack-url URL] the tag to pin for the next RC of a cloud line: the tag on + its OSS line's tip, after a new OSS RC when the tip has none + oss.py promote --rc vX.Y.Z-rc.N [--force --reason R] promote the OSS RC the cloud RC pins; prints the stable tag +retire(): called by cloud Retire for each cloud line it retires. + +cut, rc and promote print one OSS tag on stdout. Every write is re-runnable: a tag already on the same commit is +skipped, a tag on another commit fails. A branch and its new tag go in one atomic push. +""" + +import argparse +import os +import re +import subprocess +import sys + +import backport as bp +import relmgr as rm +import relstate as rs + +OSS = rs.PINS or "Zipstack/unstract" # the OSS repo the cloud lines pin +PREFIX = rs.REPOS[OSS.split("/")[1].lower()][5] +VER = rs.ver_re(PREFIX) +PIN_RE = re.compile(r'^unstract-platform-version = "([^"]*)"$', re.M) +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def pin(repo, ref): + """The OSS tag that ref's pyproject.toml pins.""" + m = PIN_RE.search(rm.git(repo, "show", f"{ref}:pyproject.toml")) + if not m: + raise RuntimeError(f"no unstract-platform-version pin in {ref}:pyproject.toml") + return m[1] + + +def line_of(tag): + m = VER.fullmatch(tag) + if not m: + raise RuntimeError( + f"OSS pin {tag!r} is not a {PREFIX}vX.Y.Z or {PREFIX}vX.Y.Z-rc.N tag" + ) + return f"release/{m[1]}.{m[2]}" + + +def tip(oss, branch): + r = rm.git( + oss, + "rev-parse", + "-q", + "--verify", + f"refs/remotes/origin/{branch}^{{commit}}", + check=False, + ) + return r.stdout.strip() if r.returncode == 0 else None + + +def tags_on(oss, sha, branch): + """Version tags of branch's minor on commit sha.""" + line = branch.removeprefix("release/") + return [ + t + for t in rm.git( + oss, "tag", "--points-at", sha, "-l", f"{PREFIX}v{line}.*" + ).split() + if (m := VER.fullmatch(t)) and f"{m[1]}.{m[2]}" == line + ] + + +def choose(tags): + """The tag to pin from one commit's tags: its stable tag, else its newest RC, else None.""" + stable = [t for t in tags if "-rc." not in t] + return max(stable or tags, key=rs._vkey) if tags else None + + +def _bot(oss, *args): + return rm.git( + oss, "-c", "user.name=releases-bot", "-c", f"user.email={rm.BOT_EMAIL}", *args + ) + + +def _tag(oss, name, sha, *msgs): + """Create tag name on sha locally (annotated when msgs); False when it is already there, an error when it is on + another commit. + """ + have = rs._tag_commit(oss, name) + if have and have != sha: + raise RuntimeError(f"OSS tag {name} is on {have[:12]}, not on {sha[:12]}") + if have: + return False + _bot( + oss, + "tag", + *(["-a", name, *(x for m in msgs for x in ("-m", m))] if msgs else [name]), + sha, + ) + return True + + +def _push(oss, *refspecs): + rm.git(oss, "push", "-q", "--atomic", "origin", *refspecs) + + +def _bump(oss, branch, base, tag, slack_url): + """One empty bot commit `release: ` on base, tagged, pushed as branch's new tip with its tag. The commit gives + the RC its own commit, so a line never shares a tag's commit with main or another line. + """ + sha = _bot( + oss, + "commit-tree", + f"{base}^{{tree}}", + "-p", + base, + "-m", + f"release: {tag.removeprefix('v')}", + ) + _tag(oss, tag, sha, *([f"Slack: {slack_url}"] if slack_url else [])) + _push(oss, f"{sha}:refs/heads/{branch}", f"refs/tags/{tag}") + rm.git(oss, "update-ref", f"refs/remotes/origin/{branch}", sha) + return sha + + +def gate(oss, branch, ref, annotate): + """The OSS-line convergence gate of branch at tag ref, run on the OSS clone as the OSS repo: 0 pass, 2 block, + 1 error. Its checks table goes to the step summary. argparse also exits 2, on a usage error: that is an error, + never a block a force could override. + """ + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(summary, "a") as f: + f.write(f"#### OSS line `{branch}` at `{ref}`\n\n") + env = { + **os.environ, + "GITHUB_REPOSITORY": OSS, + "GH_TOKEN": os.environ.get("OSS_TOKEN") or os.environ.get("GH_TOKEN", ""), + } + r = subprocess.run( + [ + sys.executable, + os.path.join(HERE, "convgate.py"), + "--repo", + oss, + "--branch", + branch, + "--target", + "oss", + "--promote-ref", + ref, + "--annotate", + annotate, + ], + env=env, + stdout=2, + stderr=subprocess.PIPE, + text=True, + ) # stdout is the tag the workflow captures + sys.stderr.write(r.stderr) + if ( + r.returncode not in (0, 2) + or r.returncode == 2 + and re.search(r"^usage: ", r.stderr, re.M) + ): + return 1 + return r.returncode + + +def cut(oss, line, slack_url=""): + """Cut OSS release/ from OSS main's tip plus its first RC. Refused when the OSS line exists, unless it is + this cut's own result from an earlier run (its tip the bot bump carrying ee-v.0-rc.1), or when the minor + has OSS tags left (a retired line that shipped). + """ + branch, tag = f"release/{line}", f"{PREFIX}v{line}.0-rc.1" + if have := tip(oss, branch): + if rs._tag_commit(oss, tag) == have and rm.already_bumped( + oss, f"refs/remotes/origin/{branch}", tag + ): + print( + f"OSS {branch} already cut at {tag} ({have[:12]}): re-run of the same cut", + file=sys.stderr, + ) + return tag + raise RuntimeError(f"OSS {branch} already exists: a cut never reuses an OSS line") + if used := [ + t + for t in rm.git(oss, "tag", "-l", f"{PREFIX}v{line}.*").split() + if VER.fullmatch(t) + ]: + raise RuntimeError( + f"OSS minor {line} already has tags ({', '.join(sorted(used, key=rs._vkey))})" + ) + _bump( + oss, branch, rm.git(oss, "rev-parse", "refs/remotes/origin/main"), tag, slack_url + ) + return tag + + +def rc(oss, branch, slack_url=""): + """The OSS tag for the next RC of cloud branch: its OSS line's tip tag, after a new OSS RC on an untagged tip. + Previews the OSS-line gate (never blocks). + """ + sha = tip(oss, branch) + if not sha: + raise RuntimeError(f"OSS {branch} does not exist: cloud Cut makes it") + if not (tag := choose(tags_on(oss, sha, branch))): + tag = rm.rc_version(oss, f"refs/remotes/origin/{branch}", PREFIX) + _bump(oss, branch, sha, tag, slack_url) + print(f"OSS {branch}: new RC {tag}", file=sys.stderr) + if "-rc." in tag and gate(oss, branch, tag, "warning") == 1: + print( + f"::warning::OSS convergence gate for {branch} did not run (see the log)", + file=sys.stderr, + ) + return tag + + +def promote(repo, oss, cloud_rc, force=False, reason=""): + """The stable OSS tag of the OSS RC cloud_rc pins, tagging it when it has none yet. Open backport conflict PRs on + the OSS line (or a failed listing of them) and its gate block first unless force; the other cloud product finds + the tag and skips both. + """ + pinned = pin(repo, cloud_rc) + branch = line_of(pinned) + if "-rc." not in pinned: + return pinned + sha = rs._tag_commit(oss, pinned) + if not sha: + raise RuntimeError(f"no OSS tag {pinned}") + try: + stable = rm.promote_version(oss, pinned) + except ValueError as e: # downgrade + raise RuntimeError(f"OSS {e}") from None + if rs._tag_commit(oss, stable) == sha: + return stable # promoted earlier, e.g. by the other cloud product + try: + prs, unchecked = ( + [ + f"#{p['number']}" + for p in bp.conflict_prs(OSS, branch, os.environ.get("OSS_TOKEN")) + ], + [], + ) + except RuntimeError as e: + if not force: + raise + print( + f"::warning::Could not list open backport conflict PRs on OSS {branch} (forced on): {e}", + file=sys.stderr, + ) + prs, unchecked = [], ["conflict PRs unchecked: listing failed"] + if prs and not force: + raise RuntimeError( + f"OSS {branch} has open backport conflict PRs {', '.join(prs)}: resolve and merge or close " + "them, or force with a reason" + ) + code = gate(oss, branch, pinned, "error") + if code == 1 or code == 2 and not force: + raise RuntimeError( + f"OSS convergence gate for {branch} at {pinned} " + + ( + "did not run (see the log)" + if code == 1 + else "blocks: MISSING picks (summary above); force with a reason" + ) + ) + forced = ( + ([f"forced: {reason}"] if code == 2 or prs or unchecked else []) + + unchecked + + ([f"open conflict PRs: {', '.join(prs)}"] if prs else []) + ) + _tag(oss, stable, sha, f"promote: {pinned} -> {stable} (cloud {cloud_rc})", *forced) + _push(oss, f"refs/tags/{stable}") + return stable + + +def retire(oss, branch, reason, dry_run=True): + """Retire OSS release/X.Y with its cloud line: archive tag retired/X.Y-, then (no OSS GA) its RC tags, + then the branch. With a GA its tags stay and the minor is never reused; without one it is. Summary lines. + """ + line = branch.removeprefix("release/") + sha = tip(oss, branch) + if not sha: + done = rm.git(oss, "tag", "-l", f"retired/{line}-*").split() + return [ + f"- OSS `{branch}`: no branch" + + (f" (archived as `{done[-1]}`)" if done else "") + ] + archive = f"retired/{line}-{sha[:7]}" + if moved := rs.moved_archive(oss, line, sha): + raise RuntimeError( + f"OSS {branch} moved to {sha[:12]} after {moved} archived it; check the new commits, then " + f"delete {moved} to retire it again" + ) + stable, _ = rm._tags(oss, *line.split("."), PREFIX) + rcs = ( + [] + if stable + else sorted( + ( + t + for t in rm.git(oss, "tag", "-l", f"{PREFIX}v{line}.*-rc.*").split() + if VER.fullmatch(t) + ), + key=rs._vkey, + ) + ) + out = [ + f"- OSS `{branch}` (tip `{sha[:12]}`; " + + ("has GA, tags kept)" if stable else "no GA, RC tags deleted)"), + *(f"- OSS git tag `{t}`" for t in rcs), + f"- OSS tag `{archive}`: {reason}", + ] + if dry_run: + return out + if _tag(oss, archive, sha, f"retired {branch}: {reason}"): + _push(oss, f"refs/tags/{archive}") + if rcs: + _push(oss, *(f":refs/tags/{t}" for t in rcs)) + _push(oss, f":refs/heads/{branch}") + return out + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--oss-repo", + default=os.environ.get("OSS_REPO"), + help="the OSS clone; default: $OSS_REPO", + ) + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("cut") + c.add_argument("--line", required=True) + c.add_argument("--slack-url", default="") + r = sub.add_parser("rc") + r.add_argument("--branch", required=True) + r.add_argument("--slack-url", default="") + m = sub.add_parser("promote") + m.add_argument("--rc", required=True) + m.add_argument("--force", action="store_true") + m.add_argument("--reason", default="") + a = p.parse_args(argv) + try: + if not a.oss_repo: + raise RuntimeError("--oss-repo (or OSS_REPO) is required") + tag = ( + cut(a.oss_repo, a.line, a.slack_url) + if a.cmd == "cut" + else rc(a.oss_repo, a.branch, a.slack_url) + if a.cmd == "rc" + else promote(".", a.oss_repo, a.rc, a.force, a.reason) + ) + except RuntimeError as e: + print(f"::error::{e}", file=sys.stderr) # stdout is the tag the workflow captures + return 1 + print(tag) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py new file mode 100644 index 0000000000..6b00a49c73 --- /dev/null +++ b/.github/release/relmgr.py @@ -0,0 +1,634 @@ +"""Pure-git checks for the release-branch strategy. + +Every function reads a local repo only; nothing is stored, everything is derived from git. +""" + +import ast +import os +import re +import subprocess + +BOT_EMAIL = "releases@unstract.com" +PICK_RE = re.compile(r"\(cherry picked from commit ([0-9a-f]{7,40})\)") +# Anywhere in the message, at a line start; `* ` too, because a squash with COMMIT_MESSAGES lists each commit's +# subject as `* ` and the bot's Release-branch-only commit is subject-only. +RBO_RE = re.compile(r"^(?:\* )?Release-branch-only: (\S+)", re.M) +# `UN-1 [FIX] x` or `[FIX] x`: the ticket is optional. +PR_TITLE_RE = re.compile( + r"^(?:(?P[A-Z][A-Z0-9]*-\d+) )?\[(?PFIX|HOT-FIX|FEAT|GATED-FEAT|DOCS|DEPS|MISC|DEV|WIP)\] \S" +) +FIX_CLASSES = ("fix", "security", "critical") +REVERT_RE = re.compile( + r'^Revert "(.+)"' +) # GitHub's revert button; greedy, so a revert of a revert nests +REVERTS_RE = re.compile( + r"^This reverts commit ([0-9a-f]{7,40})", re.M +) # git revert's body line +DECLINED_RE = re.compile( + r"^Declined: (.+)$", re.M +) # the reason line Backport PR writes on a decline commit +BACKPORT_HEAD_RE = re.compile( + r"backport/(\d+\.\d+)/([0-9a-f]{7})" +) # the head Backport PR opens: line, sha7; fullmatch + + +def _web(): + """https://github.com// inside Actions; None outside it (the links below are then plain text).""" + server, slug = ( + os.environ.get("GITHUB_SERVER_URL"), + os.environ.get("GITHUB_REPOSITORY"), + ) + return f"{server}/{slug}" if server and slug else None + + +def pr_link(n): + return f"[#{n}]({_web()}/pull/{n})" if _web() else f"#{n}" + + +def commit_link(sha, n=7): + return f"[`{sha[:n]}`]({_web()}/commit/{sha})" if _web() else f"`{sha[:n]}`" + + +def tag_link(tag): + # The release page: it shows the tag message (an RC's Slack thread) and later the published notes. + return f"[`{tag}`]({_web()}/releases/tag/{tag})" if _web() else f"`{tag}`" + + +def run_link(run_id): + return f"[run]({_web()}/actions/runs/{run_id})" if _web() else f"run {run_id}" + + +def image_link(target, tag): + """The GHCR package page of //release, the tag in the text.""" + text = f"`{target}/release:{tag}`" + if not _web(): + return text + owner, repo = os.environ["GITHUB_REPOSITORY"].split("/") + return ( + f"[{text}]({os.environ['GITHUB_SERVER_URL']}/orgs/{owner}/packages/container/package/" + f"{repo.lower()}%2F{target}%2Frelease)" + ) + + +_FULL = ( + set() +) # repos confirmed non-shallow; a failed check is retried, never cached as a pass + + +def _require_full_history(repo): + if repo in _FULL: + return + r = subprocess.run( + ["git", "-C", repo, "rev-parse", "--is-shallow-repository"], + capture_output=True, + text=True, + ) + if ( + r.stdout.strip() == "true" + ): # truncated history reads as "not an ancestor" and allocates taken names + raise RuntimeError( + f"{repo} is a shallow clone: relmgr needs full history and tags (fetch-depth: 0)" + ) + if r.returncode == 0 and r.stdout.strip() == "false": + _FULL.add(repo) + + +def git(repo, *args, check=True): + _require_full_history(repo) + r = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True) + if check and r.returncode: + raise RuntimeError( + f"git {' '.join(args)} (exit {r.returncode}): {r.stderr.strip()}" + ) + return r.stdout.strip() if check else r + + +def is_ancestor(repo, a, b): + r = git(repo, "merge-base", "--is-ancestor", a, b, check=False) + if r.returncode not in (0, 1): # 1 = "no"; anything else is an error, never a "no" + raise RuntimeError(f"git merge-base --is-ancestor {a} {b}: {r.stderr.strip()}") + return r.returncode == 0 + + +def on_main(repo, sha, main="main"): + """Does `sha` (full or unambiguous prefix) name a commit on `main`?""" + r = git(repo, "rev-parse", "-q", "--verify", f"{sha}^{{commit}}", check=False) + return r.returncode == 0 and is_ancestor(repo, r.stdout.strip(), main) + + +def _is_empty(repo, sha): + return git(repo, "rev-parse", f"{sha}^{{tree}}") == git( + repo, "rev-parse", f"{sha}^^{{tree}}" + ) + + +def picked_from(repo, sha): + """Main originals named by the commit's `-x` trailers.""" + return PICK_RE.findall(git(repo, "log", "-1", "--format=%B", sha)) + + +def branch_range(repo, branch, main): + """Commits in merge-base(main, branch)..branch: the convergence scan range.""" + base = git(repo, "merge-base", main, branch) + out = git(repo, "rev-list", "--reverse", f"{base}..{branch}") + return out.split() if out else [] + + +def range_messages(repo, branch, main): + """All commit messages in merge-base(main, branch)..branch, as one string (one git call).""" + return git( + repo, "log", "--format=%B", f"{git(repo, 'merge-base', main, branch)}..{branch}" + ) + + +# 1. Commit kinds / release-branch CI gate --------------------------------------------------------------- + + +def reverted_pick(repo, sha, main="main"): + """The full sha of the pick that `sha` reverts on its own branch (after the cut), else None.""" + m = REVERTS_RE.search(git(repo, "log", "-1", "--format=%B", sha)) + if not m or git(repo, "cat-file", "-e", f"{m[1]}^{{commit}}", check=False).returncode: + return None + t = git(repo, "rev-parse", f"{m[1]}^{{commit}}") + on_branch = is_ancestor(repo, t, sha) and not is_ancestor(repo, t, main) + return ( + t if on_branch and classify_commit(repo, t, main=main) == "cherry-pick" else None + ) + + +def classify_commit(repo, sha, bot_email=BOT_EMAIL, main="main"): + body = git(repo, "log", "-1", "--format=%B", sha) + if PICK_RE.search(body): + if not all(on_main(repo, o, main) for o in PICK_RE.findall(body)): + return ( + "invalid" # trailer names nothing on main: not a pick of a main original + ) + return "declined" if _is_empty(repo, sha) else "cherry-pick" + if reverted_pick( + repo, sha, main + ): # backing a pick out of one branch declines it there + return "declined" + if RBO_RE.search( + body + ): # before the author check: a bot-made pin bump is still owed a report + return "release-branch-only" + # Trust assumption: the author email is client-set; the review every release-branch PR gets is the backstop. + if git(repo, "log", "-1", "--format=%ae", sha).lower() == bot_email.lower(): + return "build-generated" + return "invalid" + + +def pr_commits(repo, base, head_ref, head="HEAD"): + """The commits the PR check classifies on a PR from head_ref into base. A backport/* PR lands as its squash, whose + message keeps the first commit's pick trailer: only that commit is judged, so a plain resolution commit after it + passes. Any other PR: every commit in base..head. + """ + out = git(repo, "rev-list", "--reverse", f"{base}..{head}") + commits = out.split() if out else [] + return commits[:1] if is_backport(head_ref) else commits + + +def is_backport(head_ref): + """Is head_ref a branch Backport PR names (backport/X.Y/)? Any other name is judged commit by commit.""" + return bool(BACKPORT_HEAD_RE.fullmatch(head_ref)) + + +def pick_trailer(head_ref, msg): + """The one pick trailer of msg when it names head_ref's sha7 (backport/X.Y/), else None.""" + found = PICK_RE.findall(msg) + m = BACKPORT_HEAD_RE.fullmatch(head_ref) + return found[0] if m and len(found) == 1 and found[0].startswith(m[2]) else None + + +def conflict_markers(repo, rev="HEAD"): + """Files in rev's tree with a `<<<<<<<` or `>>>>>>>` conflict-marker line.""" + r = git(repo, "grep", "-l", "-E", "^(<<<<<<<|>>>>>>>)( |$)", rev, "--", check=False) + if r.returncode not in (0, 1): # 1 = no match + raise RuntimeError(f"git grep (exit {r.returncode}): {r.stderr.strip()}") + return [m.split(":", 1)[1] for m in r.stdout.splitlines()] + + +def ci_commits( + repo, branch, event, before="", head_ref="", labelled=False, main="origin/main" +): + """Release branch CI over one event: ([(sha, kind)], where). ValueError: conflict markers in a backport PR's tree. + + pull_request: the PR's own commits (a backport PR's first only, invalid unless it carries exactly one pick + trailer, naming the head's sha7); labelled turns its invalid commits into + release-branch-only, since the squash at merge writes the line. push: the pushed commits before..HEAD, or the + whole branch when before is all zeros (branch creation) or gone (a force push). + """ + if event == "pull_request": + if is_backport(head_ref) and (marked := conflict_markers(repo)): + raise ValueError( + f"conflict markers in {', '.join(marked)}: resolve before merging" + ) + commits, where = pr_commits(repo, branch, head_ref), f"PR into {branch}" + elif ( + set(before) - {"0"} + and git(repo, "cat-file", "-e", f"{before}^{{commit}}", check=False).returncode + == 0 + ): + # Only the pushed commits. Classifying the whole branch would keep every later push red + # after one invalid commit landed, so a revert could never clear it. The PR check covers PR commits. + out = git(repo, "rev-list", "--reverse", f"{before}..HEAD") + commits, where = out.split() if out else [], f"push to {branch}" + else: + commits, where = branch_range(repo, branch, main), branch + labelled = labelled and event == "pull_request" + rows = [] + for c in commits: + kind = classify_commit(repo, c, main=main) + # The squash keeps every trailer of a backport PR's first commit: one naming the head's fix, or it records others. + if ( + event == "pull_request" + and is_backport(head_ref) + and not pick_trailer(head_ref, git(repo, "log", "-1", "--format=%B", c)) + ): + kind = "invalid" + rows.append( + (c, "release-branch-only (label)" if labelled and kind == "invalid" else kind) + ) + return rows, where + + +def check_branch(repo, branch, main="main"): + """Release-branch CI gate: the invalid commits on the branch. Empty list == pass.""" + return [ + c + for c in branch_range(repo, branch, main) + if classify_commit(repo, c, main=main) == "invalid" + ] + + +# 2. PR class (Lint PR) ------------------------------------------------------ + +FIX_LABELS = ("fix/security", "fix/critical", "fix/needs-backport") + + +def pr_fix_class(title, labels=()): + """'fix' | 'security' | 'critical' for fix PRs, None for non-fix PRs; ValueError = CI rejects.""" + # Anything that looks like a class label (fix/, the old fix-class/, near-typos) but is + # not one of FIX_LABELS: a typo must not silently drop or downgrade the class. + bad = [ + l + for l in labels + if l not in FIX_LABELS + and re.match(r"\s*fix(\w{0,3}[\W_]*class(?![a-z])|\s*/)", l, re.I) + ] + if bad: + raise ValueError(f"unknown fix label(s) {bad}; allowed: {', '.join(FIX_LABELS)}") + classes = { + l.split("/", 1)[1] for l in labels if l in ("fix/security", "fix/critical") + } + if len(classes) > 1: + raise ValueError(f"conflicting fix labels: {sorted('fix/' + c for c in classes)}") + # A revert takes the reverted PR's class: it must follow the original onto every branch that carries it. + inner = title + while r := REVERT_RE.match(inner): + inner = r.group(1) + m = PR_TITLE_RE.match(inner) + if not m and not set(labels) & set(FIX_LABELS): + raise ValueError( + f"no recognised '[TYPE]' or 'TICKET [TYPE]' prefix and no fix/ label: {title!r}" + ) + if classes: + return classes.pop() + # fix/needs-backport: a non-[FIX] PR (e.g. [MISC]) that stabilizing lines need; no-op on a [FIX] title. + return ( + "fix" + if (m and m["type"] in ("FIX", "HOT-FIX")) or "fix/needs-backport" in labels + else None + ) + + +def rbo_key(title, pr): + """The `Release-branch-only:` value for PR `pr`: the title's ticket, else `#`. A `#` key matches no + other line's, so the convergence report always lists that change on the newer line for a human to check. + """ + m = PR_TITLE_RE.match(title) + return m["ticket"] if m and m["ticket"] else f"#{pr}" + + +# 3. Version allocation (at build time; a version names one commit) ------------------------------------------------ + + +def _line(branch): + m = re.fullmatch( + r"(?:refs/heads/|(?:refs/remotes/)?[^/]+/)?release/(\d+)\.(\d+)", branch + ) + if not m: + raise ValueError(f"not a release branch: {branch}") + return m.group(1), m.group(2) + + +def _tags(repo, x, y, prefix=""): + """({stable Z}, {Z: [RC N]}) of line x.y. prefix: the repo's tag prefix (`ee-` on OSS lines); tags without it are + another series and never counted. + """ + stable, rc = set(), {} + for t in git(repo, "tag", "-l", f"{prefix}v{x}.{y}.*").split(): + m = re.fullmatch(rf"{re.escape(prefix)}v{x}\.{y}\.(\d+)(?:-rc\.(\d+))?", t) + if m and m.group(2): + rc.setdefault(int(m.group(1)), []).append(int(m.group(2))) + elif m: + stable.add(int(m.group(1))) + return stable, rc + + +def next_patch(repo, branch, prefix=""): + """The X.Y.Z RC builds are cut for: one past the highest stable Z (0 before GA).""" + x, y = _line(branch) + stable, _ = _tags(repo, x, y, prefix) + return f"{x}.{y}.{max(stable) + 1 if stable else 0}" + + +def rc_version(repo, branch, prefix="", taken=()): + """Tag for a build of the branch tip, allocated from tags at build time. + + If the tip already carries an RC tag of this line, that tag is returned: a commit gets one RC tag. + taken: tag names already used elsewhere (every target's registry tags). An RC there is never allocated again: + an artifact pushed by a run whose tag push failed would otherwise block the line. The one exception is a tip + that is already the bot's bump for it (already_bumped): that run is resumed. + Needs the full tag set locally (fetch tags in CI): with no tags every line reads as freshly cut. + """ + x, y = _line(branch) + for t in git(repo, "tag", "--points-at", branch).split(): + if re.fullmatch(rf"{re.escape(prefix)}v{x}\.{y}\.\d+-rc\.\d+", t): + return t + ver = next_patch(repo, branch, prefix) + _, rc = _tags(repo, x, y, prefix) + tagged = max(rc.get(int(ver.rsplit(".", 1)[1]), [0])) + pushed = [ + int(m[1]) + for t in taken + if (m := re.fullmatch(rf"{re.escape(prefix)}v{re.escape(ver)}-rc\.(\d+)", t)) + ] + top = max([tagged, *pushed]) + for n in range(tagged + 1, top + 1): + if already_bumped(repo, branch, f"{prefix}v{ver}-rc.{n}"): + return f"{prefix}v{ver}-rc.{n}" + return f"{prefix}v{ver}-rc.{top + 1}" + + +CHART = "charts/unstract-platform/Chart.yaml" + + +def already_bumped(repo, branch, tag): + """True when the branch tip is the bot's bump for tag: build-generated with subject `release: chart ` + and the chart at the version, or, in a repo with no chart, subject `release: ` (an empty commit). + That tip is left behind when the bump was pushed but the run stopped before the tag push (a failed build); + Take RC build then tags it instead of committing another bump. + """ + ver = tag.removeprefix("v") + chart = git(repo, "show", f"{branch}:{CHART}", check=False) + if chart.returncode == 0 and not re.search( + rf"^version: {re.escape(ver)}$", chart.stdout, re.M + ): + return False + email, subject = git(repo, "log", "-1", "--format=%ae%n%s", branch).split("\n", 1) + want = f"release: chart {ver}" if chart.returncode == 0 else f"release: {ver}" + return email.lower() == BOT_EMAIL.lower() and subject == want + + +def promote_version(repo, rc_tag): + """Stable tag for promoting rc_tag: the commit's existing stable tag, else one past the highest. + + Stable Z follows commit order, so an upgrade never loses a fix. The Z can differ from the + RC's (a target that passes a later RC takes the next patch, so a version names one commit; the + RC->stable mapping goes in the release body), which departs from the spec's + x.y.(z+1)-rc.N -> vx.y.(z+1) when a promote lands while a later patch is in RC. An RC older than a + commit that already has a stable tag is refused as a downgrade. Both are harness choices, open + with the spec owner. + """ + m = re.fullmatch( + r"((?:[a-z]+-)?)v(\d+)\.(\d+)\.(\d+)-rc\.\d+", rc_tag + ) # the stable tag keeps the RC's prefix + if not m: + raise ValueError(f"not an RC tag: {rc_tag}") + p, x, y = m.group(1), m.group(2), m.group(3) + commit = git(repo, "rev-parse", f"{rc_tag}^{{commit}}") + stable, _ = _tags(repo, x, y, p) + for z in sorted(stable): + shipped = git(repo, "rev-parse", f"refs/tags/{p}v{x}.{y}.{z}^{{commit}}") + if shipped == commit: + return f"{p}v{x}.{y}.{z}" + if is_ancestor(repo, commit, shipped): + raise ValueError( + f"{rc_tag} is older than {p}v{x}.{y}.{z}: promoting it would be a downgrade" + ) + return f"{p}v{x}.{y}.{max(stable) + 1 if stable else 0}" + + +# 4. Convergence -------------------------------------------------------------------- + + +def is_converged(repo, c_main, branch, main="main"): + """Inherited (ancestor) or picked/declined after the cut (trailer in merge-base..branch).""" + full = git(repo, "rev-parse", f"{c_main}^{{commit}}") + if is_ancestor(repo, full, branch): + return True + return any( + o.startswith(full) or full.startswith(o) + for o in PICK_RE.findall(range_messages(repo, branch, main)) + ) + + +def trailer_states(repo, branch, main="main"): + """{original: "cherry-pick" | "declined"} from merge-base(main, branch)..branch, the newest commit per original + winning: a pick after a decline ships the fix again (a decline undone by merging the reopened PR), a revert of + a pick declines it. Declines are empty pick-trailer commits and reverts of picks. + """ + state = {} + for c in branch_range(repo, branch, main): + kind = classify_commit(repo, c, main=main) + if kind in ("cherry-pick", "declined"): + for o in picked_from(repo, reverted_pick(repo, c, main) or c): + state[o] = kind + return state + + +def declined_originals(repo, branch, main="main"): + """Originals whose newest pick-trailer commit after the cut is a decline (see trailer_states).""" + return { + o for o, kind in trailer_states(repo, branch, main).items() if kind == "declined" + } + + +def decline_records(repo, branch, main="main"): + """{original: (decline sha, reason)} for the originals declined_originals returns, from the same walk. Reason: the + last `Declined: ...` line of an empty decline commit (its subject if it has none), or a revert's subject. + """ + rec = {} + for c in branch_range(repo, branch, main): + kind = classify_commit(repo, c, main=main) + if kind in ("cherry-pick", "declined"): + rp = reverted_pick(repo, c, main) + body = git(repo, "log", "-1", "--format=%B", c) + reasons = DECLINED_RE.findall(body) + reason = reasons[-1].strip() if reasons and not rp else body.splitlines()[0] + for o in picked_from(repo, rp or c): + rec[o] = (c, reason) if kind == "declined" else None + return {o: r for o, r in rec.items() if r} + + +def shipped_prs(repo, tags, main="main", lookup=None, line=None): + """PR numbers whose main original was cherry-picked (declines excluded) into any of `tags`: the changelog drops them. + + Newest trailer wins per tag (see trailer_states): a pick that a later decline or revert on that line backed out is + not shipped, and a pick after a decline is. The PR number is the `(#n)` at the end of the original's subject + (every squash on main has it); `lookup(sha) -> int | None` is the fallback for an original without one. + + line: the ref being published (Publish passes its RC tag). Its picks of a shipped original are dropped too, by the + PR that landed them: `lookup(pick)` (a clean pick keeps the original's subject, so its `(#n)` is the main PR) and + the pick's own `(#n)` (a conflict pick lands as the PR's squash). A pick with neither is skipped, not an error. + """ + originals = { + o + for t in tags + for o, kind in trailer_states(repo, t, main).items() + if kind == "cherry-pick" + } + prs = set() + for o in originals: + m = re.search(r"\(#(\d+)\)$", git(repo, "log", "-1", "--format=%s", o)) + n = int(m[1]) if m else lookup(o) if lookup else None + if n is None: + raise RuntimeError( + f"no PR number for {o[:12]} (picked into {', '.join(tags)})" + ) + prs.add(n) + for c in branch_range(repo, line, main) if line else []: + if classify_commit(repo, c, main=main) == "cherry-pick" and any( + o.startswith(d) or d.startswith(o) + for o in originals + for d in picked_from(repo, c) + ): # may be abbreviated + m = re.search(r"\(#(\d+)\)$", git(repo, "log", "-1", "--format=%s", c)) + prs |= { + n + for n in (int(m[1]) if m else None, lookup(c) if lookup else None) + if n is not None + } + return prs + + +def convergence_report(repo, older, newer, main="main", newer_branch=None): + """[(sha_on_older, status)] with status CONVERGED | MISSING | UNCLASSIFIABLE | RBO-ABSENT. Blocks iff any MISSING. + + RBO-ABSENT is report-only: a Release-branch-only key (the ticket, or `#`) on `older` with no + Release-branch-only commit for that key on `newer`. Case 5 (config/migration) owes nothing; + a security fix for code gone from main may need its own equivalent on `newer` if the vulnerable code is still + there, so a human looks. + + Both scan directions are this one call: promoting `newer` scans each older branch into it, + promoting `older` scans it into each newer branch; either way it is older's picks vs newer. + Pass `newer` as a branch while it is in RC. Once it is GA, pass its latest stable tag (for + the promoting target) as `newer`, since "shipped in its latest patch" is not the branch tip, and + the branch as `newer_branch`: a branch need only carry a decline, not ship it. + + A pick whose original cannot be placed on `main` fails release-branch CI and is MISSING here: it is never + report-only, and never CONVERGED (a branch-first fix has no main original to converge on). + """ + out = [] + # Newest trailer wins on both sides: a decline on newer that a later pick undid is not a decline, and a pick on + # older that a later decline or revert backed out is not owed. + declined = declined_originals(repo, newer_branch or newer, main) + newer_picks = { + o + for o, kind in trailer_states(repo, newer, main).items() + if kind == "cherry-pick" + } + older_declined = declined_originals(repo, older, main) + newer_rbo = set(RBO_RE.findall(range_messages(repo, newer, main))) + same = lambda o, names: any( + o.startswith(d) or d.startswith(o) for d in names + ) # trailers may be abbreviated + for c in branch_range(repo, older, main): + kind = classify_commit(repo, c, main=main) + if kind == "invalid": + out.append((c, "MISSING" if picked_from(repo, c) else "UNCLASSIFIABLE")) + elif kind == "cherry-pick": + if all( + same(o, older_declined) for o in picked_from(repo, c) + ): # backed out of `older`: newer owes nothing + continue + ok = all( + is_ancestor(repo, git(repo, "rev-parse", f"{o}^{{commit}}"), newer) + or same(o, newer_picks) + or same(o, declined) + for o in picked_from(repo, c) + ) + out.append((c, "CONVERGED" if ok else "MISSING")) + elif kind == "release-branch-only": + body = git(repo, "log", "-1", "--format=%B", c) + if not set(RBO_RE.findall(body)) & newer_rbo: + out.append((c, "RBO-ABSENT")) + # build-generated: excluded from convergence. + # declined on `older`: older does not carry the fix, so newer owes nothing. + return out + + +# 5. Relevance (advisory: does the branch still have the code a fix touches?) ------------------ + + +def relevance(repo, fix_sha, branch): + """Are the exact line versions a main fix edits on `branch`? relevant | not-relevant | partial | unknown. + + Blames the pre-image lines the fix modifies/deletes in fix^; each last-modifying commit counts + if it is an ancestor of `branch` or was picked onto it. 'not-relevant' means those line versions + are absent, not that the code is: a line rewritten on main after the cut still exists on the + branch in its older form (adapt the pick by hand, never refactor). Advisory only. + Pure additions have no pre-image: 'unknown' (fallback, not implemented: treat as relevant iff + the touched files exist on the branch). + """ + parent, introducers, path, header = f"{fix_sha}^", set(), None, False + diff = git( + repo, + "-c", + "core.quotePath=false", + "diff", + "-U0", + "--no-renames", + "--src-prefix=a/", + "--dst-prefix=b/", + parent, + fix_sha, + ) + for line in diff.splitlines(): + if line.startswith("diff --git "): + path, header = None, True + elif header and line.startswith( + "--- " + ): # only in the file header: `--- x` in a hunk is content + p = line[4:].rstrip("\t") # git tab-suffixes paths with spaces + if p.startswith( + '"' + ): # C-quoted (", \, control chars): unquote to the raw bytes + p = ast.literal_eval("b" + p).decode() + path = None if p == "/dev/null" else p[2:] + elif line.startswith("@@"): + header = False + m = re.match(r"@@ -(\d+)(?:,(\d+))? ", line) + if m and path and m.group(2) != "0": + start, n = int(m.group(1)), int(m.group(2) or 1) + blame = git( + repo, "blame", "--porcelain", "-L", f"{start},+{n}", parent, "--", path + ) + introducers |= set(re.findall(r"^([0-9a-f]{40}) \d+ \d+", blame, re.M)) + if not introducers: + return "unknown" + # ponytail: scans the whole branch log once per call; fine for fixtures, index trailers once if slow. + picked = { + o + for c in git( + repo, "log", "--format=%H", "-F", "--grep=(cherry picked from commit", branch + ).split() + if not _is_empty(repo, c) + for o in picked_from(repo, c) + } # a decline carries nothing + hits = [ + is_ancestor(repo, c, branch) or any(c.startswith(p) for p in picked) + for c in introducers + ] + return "relevant" if all(hits) else "not-relevant" if not any(hits) else "partial" diff --git a/.github/release/relstate.py b/.github/release/relstate.py new file mode 100644 index 0000000000..316460516e --- /dev/null +++ b/.github/release/relstate.py @@ -0,0 +1,578 @@ +"""Branch and target state for the V2 release workflows, derived from git + the target registries. + +Usage: relstate.py [--repo DIR] [--now 2026-09-29T00:00:00Z] [--rc vX.Y.Z-rc.N] + +Prints one JSON document: every release/X.Y branch with its per-target state (enabled, GA, support window), +plus the derived sets later workflows need. Nothing is stored. Version allocation, commit +classification and convergence stay in relmgr.py. +""" + +import argparse +import datetime as dt +import json +import os +import re +import subprocess +import sys +import time +import urllib.parse + +import relmgr as rm + +WINDOW = dt.timedelta( + days=184 +) # on-prem support window from the chart push date: 6 months + 1 day buffer +WINDOW_LINES = ( + 6 # and only the newest 6 on-prem GA lines are supported, whichever ends first +) +# Per repo, so both repos run this same file: (probe artifact, targets, targets where only the latest GA line is +# supported, targets supported while the branch exists: retired only when another repo's Retire names the line, +# repo whose tags this repo's lines pin in pyproject.toml or None, tag prefix of this repo's lines). +# PROBE is an artifact known to exist; it proves the token can read packages (a 404 alone is ambiguous). +# TARGETS: target -> (artifact holding RC / promoted builds, GA anchor artifact). Cloud GA = the vX.Y.Z retag +# of the build artifact; on-prem GA = the chart push. GIT instead of an artifact: the line's own git tags, the stable +# tag's date as the GA date (OSS lines: cloud builds them from source, so they have no artifact). +# OSS lines are tagged ee-vX.Y.Z(-rc.N); plain vX.Y.Z tags there are community releases from main, never a line's. +GIT = "git" +REPOS = { + "unstract-cloud": ( + "cloud/smoke", + { + "cloud": ("cloud/release", "cloud/release"), + "on-prem": ("on-prem/release", "on-prem/chart"), + }, + {"cloud"}, + set(), + "Zipstack/unstract", + "", + ), + "unstract": (None, {"oss": (GIT, GIT)}, set(), {"oss"}, None, "ee-"), +} +OWNER, PKG_PREFIX = ( + os.environ.get("GITHUB_REPOSITORY", "zipstack/unstract-cloud").lower().split("/") +) + + +def ver_re(prefix): + return re.compile(rf"{re.escape(prefix)}v(\d+)\.(\d+)\.(\d+)(?:-rc\.(\d+))?") + + +def use(slug): + """Load the REPOS config of slug (a repo name) into the module globals.""" + global PROBE, TARGETS, LATEST_ONLY, KEPT, PINS, PREFIX, VER_RE + if slug not in REPOS: + raise RuntimeError( + f"relstate has no config for repo {slug!r} (GITHUB_REPOSITORY); known: {', '.join(REPOS)}" + ) + PROBE, TARGETS, LATEST_ONLY, KEPT, PINS, PREFIX = REPOS[slug] + VER_RE = ver_re(PREFIX) + + +use(PKG_PREFIX) + + +def _gh(*args): + return subprocess.run( + ["gh", "api", "--paginate", *args], capture_output=True, text=True + ) + + +def _versions_url(artifact): + return f"/orgs/{OWNER}/packages/container/{urllib.parse.quote(f'{PKG_PREFIX}/{artifact}', safe='')}/versions?per_page=100" + + +def registry_tags(artifact): + """{tag: {digest, pushed_at}} for one GHCR artifact of this repo. Raises on any read failure. + + A 404 is only read as "artifact not pushed yet" when the token can read PROBE; otherwise it may be + a token that cannot read packages, and an empty registry would read as "nothing enabled". + """ + r = _gh( + _versions_url(artifact), + "--jq", + ".[] | {id, digest: .name, pushed_at: .created_at, tags: .metadata.container.tags}", + ) + if r.returncode: + if "HTTP 404" not in r.stderr: + raise RuntimeError(f"registry read {artifact}: {r.stderr.strip()}") + if artifact == PROBE or _gh(_versions_url(PROBE), "--jq", "length").returncode: + raise RuntimeError( + f"registry read {artifact}: 404 and probe {PROBE} unreadable: {r.stderr.strip()}" + ) + return {} + out = {} + for line in r.stdout.splitlines(): + v = json.loads(line) + for tag in v["tags"] or []: + out[tag] = {"id": v["id"], "digest": v["digest"], "pushed_at": v["pushed_at"]} + return out + + +def taken_tags(reg=None, registry=registry_tags): + """Every tag on every registry artifact of TARGETS: an RC there is never allocated again (rc_version taken). + reg: {artifact: tags} already read; artifacts missing from it are read. + """ + reg = reg or {} + return { + t + for a in sorted({a for pair in TARGETS.values() for a in pair} - {GIT}) + for t in (reg[a] if a in reg else registry(a)) + } + + +def _ts(s): + return dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + + +def _key(line): + return tuple(int(p) for p in line.split(".")) + + +def _vkey(tag): + m = re.fullmatch(r"(?:[a-z]+-)?" + ver_re("").pattern, tag) + return int(m[3]), int(m[4] or 0) + + +def _branches(repo): + """{"release/X.Y": ref}; origin's copy wins over a local one. Legacy release/vX.Y.Z heads are ignored.""" + out = {} + for ref in rm.git( + repo, + "for-each-ref", + "--format=%(refname)", + "refs/heads/release/", + "refs/remotes/origin/release/", + ).split(): + name = re.sub(r"^refs/(heads|remotes/origin)/", "", ref) + if re.fullmatch(r"release/\d+\.\d+", name) and ( + name not in out or ref.startswith("refs/remotes/") + ): + out[name] = ref + return out + + +def ahead_of_ga(repo, ref, line): + """Commits on ref after the line's latest stable git tag (what a rebuild would ship); None before GA.""" + stable, _ = rm._tags(repo, *line.split("."), PREFIX) + return ( + int(rm.git(repo, "rev-list", "--count", f"{PREFIX}v{line}.{max(stable)}..{ref}")) + if stable + else None + ) + + +def _tag_commit(repo, tag): + r = rm.git( + repo, "rev-parse", "-q", "--verify", f"refs/tags/{tag}^{{commit}}", check=False + ) + return r.stdout.strip() if r.returncode == 0 else None + + +def moved_archive(repo, line, sha): + """An archive tag retired/-* that an earlier run put on an ancestor of sha: the line moved after it was + archived, so a second tag would split its record. None otherwise. A tag off sha's history (an earlier line of a + reused minor) is not one. + """ + for t in rm.git(repo, "tag", "-l", f"retired/{line}-*").split(): + if t != f"retired/{line}-{sha[:7]}" and rm.is_ancestor( + repo, _tag_commit(repo, t), sha + ): + return t + return None + + +def git_tags(repo): + """{tag: {digest: commit, pushed_at}} of the repo's version tags: the GIT stand-in for a registry artifact.""" + out = {} + for row in rm.git( + repo, + "for-each-ref", + "--format=%(refname:short) %(creatordate:iso-strict) %(objectname) %(*objectname)", + f"refs/tags/{PREFIX}v*", + ).splitlines(): + tag, date, obj, peeled = (row.split() + [""])[:4] + if VER_RE.fullmatch(tag): + out[tag] = {"id": None, "digest": peeled or obj, "pushed_at": date} + return out + + +def rc_slack(repo, tag): + """The `Slack: ` line of an annotated RC tag's message (Take RC build slack_url), else None.""" + m = re.search( + r"^Slack: (\S+)$", rm.git(repo, "tag", "-l", tag, "--format=%(contents)"), re.M + ) + return m[1] if m else None + + +def target_state(t, name, rows, now): + """State of target t on an enabled branch. The one place the GA/supported convention lives. + + On-prem: stabilized for WINDOW after the chart push, and only while among the newest WINDOW_LINES lines with an + on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch + for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilized from its GA + while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. + """ + ga = rows[name]["targets"][t]["ga"] + if not ga: + return "stabilizing" + if t in KEPT: + return "stabilized" + t_ga = sorted( + (n for n, r in rows.items() if r["targets"].get(t, {}).get("ga")), + key=lambda n: _key(rows[n]["line"]), + reverse=True, + ) + if t in LATEST_ONLY: + return "stabilized" if name == t_ga[0] else "past-eos" + return ( + "stabilized" + if now < _ts(ga["pushed_at"]) + WINDOW and name in t_ga[:WINDOW_LINES] + else "past-eos" + ) + + +def _main(repo): + return ( + "origin/main" + if rm.git( + repo, "rev-parse", "-q", "--verify", "refs/remotes/origin/main", check=False + ).returncode + == 0 + else "main" + ) + + +def _next_minor(repo, ordered): + """One minor past the highest release/X.Y line, shipped retired line or stable vX.Y.Z on main, so a cut never + reuses a shipped minor. A line retired early before any GA (retired/X.Y-, its RCs deleted) is reused. + + With no release/X.Y branch yet (cutover) only the tags count. + """ + main = _main(repo) + lines = [ + (int(m[1]), int(m[2])) + for t in rm.git(repo, "tag", "--merged", main, "-l", f"{PREFIX}v*").split() + if (m := VER_RE.fullmatch(t)) and not m[4] + ] + [_key(r["line"]) for r in ordered] + lines += [ + (int(m[1]), int(m[2])) + for t in rm.git(repo, "tag", "-l", "retired/*").split() + if (m := re.fullmatch(r"retired/(\d+)\.(\d+)-[0-9a-f]+", t)) + and rm._tags(repo, m[1], m[2], PREFIX)[0] + ] + return f"{max(lines)[0]}.{max(lines)[1] + 1}" if lines else None + + +def derive(repo, now, registry=registry_tags, rc=None): + t0 = time.monotonic() + reg = { + a: git_tags(repo) if a == GIT else registry(a) + for a in sorted({a for pair in TARGETS.values() for a in pair}) + } + print( + f"relstate: registry read {len(reg)} artifacts in {time.monotonic() - t0:.2f}s", + file=sys.stderr, + ) # registry read cost + refs = _branches(repo) + rows, anomalies = {}, [] + taken = taken_tags(reg) + for name, ref in refs.items(): + tip = rm.rc_version( + repo, ref, PREFIX, taken + ) # the tip's RC tag if built (one RC per commit), else the next one + built = tip in rm.git(repo, "tag", "--points-at", ref).split() + rows[name] = { + "branch": name, + "line": name.split("/", 1)[1], + "state": None, + "enabled": [], + "next_rc": None if built else tip, + "tip_rc": tip if built else None, + "ahead_of_ga": ahead_of_ga(repo, ref, name.split("/", 1)[1]), + "targets": {}, + } + if built and (url := rc_slack(repo, tip)): # only when the RC has a link + rows[name]["tip_rc_slack"] = url + # A line with no branch but a git stable tag was promoted and then retired: its registry tags are + # history, not anomalies, and are skipped before any per-tag git call. A line that never + # promoted (RC-only) stays an anomaly. Retire only lists lines that were GA, so this is exact. + shipped = {} + + # enabled: a build tag of T whose git commit is on the branch (derived, never recorded). + for t, (builds, anchor) in TARGETS.items(): + for tag in sorted(reg[builds]): + m = VER_RE.fullmatch(tag) + if not m: + continue # not a release version (e.g. smoke tags) + name = f"release/{m[1]}.{m[2]}" + if name not in refs and shipped.setdefault( + name, bool(rm._tags(repo, m[1], m[2], PREFIX)[0]) + ): + continue + sha = _tag_commit(repo, tag) + if name not in refs or not sha or not rm.is_ancestor(repo, sha, refs[name]): + why = ( + "no branch" + if name not in refs + else "no git tag" + if not sha + else "commit not on branch" + ) + anomalies.append( + { + "target": t, + "artifact": builds, + "tag": tag, + "branch": name, + "reason": why, + } + ) + continue + tt = rows[name]["targets"].setdefault( + t, {"state": None, "rcs": [], "ga": None, "promoted": None} + ) + if m[4]: + tt["rcs"].append(tag) + for name, row in rows.items(): + tt = row["targets"].get(t) + if not tt: + continue + tt["rcs"].sort(key=_vkey) + stable = re.compile(rf"{re.escape(PREFIX)}v{re.escape(row['line'])}\.\d+") + # ga anchors state and the support window; promoted (the build artifact's latest stable) is what ships next + for key, artifact in (("ga", anchor), ("promoted", builds)): + if tags := sorted( + (g for g in reg[artifact] if stable.fullmatch(g)), key=_vkey + ): + tt[key] = {"tag": tags[-1], **reg[artifact][tags[-1]]} + + for name, row in rows.items(): + for t, tt in row["targets"].items(): + tt["state"] = target_state(t, name, rows, now) + row["enabled"] = sorted(row["targets"]) + states = {tt["state"] for tt in row["targets"].values()} + # No enabled target (cut, nothing built yet) is stabilizing: every line builds cloud by default. + row["state"] = ( + "stabilizing" + if not states or "stabilizing" in states + else "stabilized" + if "stabilized" in states + else "past-eos" + ) + + ordered = sorted(rows.values(), key=lambda r: _key(r["line"]), reverse=True) + out = { + "now": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "branches": ordered, + "supported": { + t: [ + r["branch"] + for r in ordered + if r["targets"].get(t, {}).get("state") in ("stabilizing", "stabilized") + ] + for t in TARGETS + }, + "backport_targets": { + "fix": [r["branch"] for r in ordered if r["state"] == "stabilizing"], + "security_critical": [ + { + "branch": r["branch"], + "needs_rm_confirmation": r["state"] == "stabilized", + } + for r in ordered + if r["state"] in ("stabilizing", "stabilized") + ], + }, + "retire": [ + r["branch"] + for r in ordered + if r["state"] == "past-eos" and any(tt["ga"] for tt in r["targets"].values()) + ], + "next_minor": _next_minor(repo, ordered), + "anomalies": anomalies, + } + if rc: + out["promote"] = {"rc": rc, "stable": rm.promote_version(repo, rc)} + return out + + +def retire_line_refusal(state, branch, force=False): + """Why Retire `retire_line` refuses branch, or None. A stabilizing target (RCs, no GA: an on-prem line never + promoted, say) may go. A stabilized on-prem target (chart in its window) goes only with force. Stabilized cloud + is production and never goes: relstate would make the next older cloud GA production again. + """ + row = next((b for b in state["branches"] if b["branch"] == branch), None) + if not row: + return "no such branch" + held = [t for t, tt in sorted(row["targets"].items()) if tt["state"] == "stabilized"] + if "cloud" in held: + return "cloud is production (newest cloud GA); it retires once a newer line is promoted" + if held and not force: + return f"{', '.join(held)} still supported (stabilized); force to retire it inside the window" + return None + + +def abandon_plan(repo, branch, registry=registry_tags): + """What Retire `abandon` deletes for an existing release/X.Y line that never shipped (cut by mistake, or built only as RCs). + + {"ga": [...], "used": [...], "rc_tags": [...], "versions": {artifact: [{id, digest, tags}]}, "kept": [...]}. + A non-empty ga (git stable tag or GA-anchor registry tag) or used (commits that are not bot, pick or + decline) refuses the abandon. A registry version is deleted only when every tag on it is an RC of + this line. GitHub refuses to delete a package's last version: such a version is kept and listed. + """ + x, y = rm._line(branch) + rc_re = re.compile(rf"{re.escape(PREFIX)}v{x}\.{y}\.\d+-rc\.\d+") + reg = { + a: registry(a) + for a in sorted({a for pair in TARGETS.values() for a in pair} - {GIT}) + } + main = _main(repo) + ga = [f"{PREFIX}v{x}.{y}.{z}" for z in sorted(rm._tags(repo, x, y, PREFIX)[0])] + ga += [ + f"{a}:{t}" + for a in sorted({anchor for _, anchor in TARGETS.values()} - {GIT}) + for t in sorted(reg[a]) + if re.fullmatch(rf"v{x}\.{y}\.\d+", t) + ] + used = [ + c[:12] + for c in rm.branch_range(repo, _branches(repo)[branch], main) + if rm.classify_commit(repo, c, main=main) + not in ("build-generated", "cherry-pick", "declined") + ] + versions, kept = {}, [] + for a, tags in reg.items(): + by_digest = {} + for t, info in sorted(tags.items()): + by_digest.setdefault( + info["digest"], + {"id": info.get("id"), "digest": info["digest"], "tags": []}, + )["tags"].append(t) + drop = [ + v for v in by_digest.values() if all(rc_re.fullmatch(t) for t in v["tags"]) + ] + # ponytail: untagged versions are invisible here, so "last" is counted over tagged ones (conservative). + if drop and len(drop) == len(by_digest): + kept.append(f"{a}:{','.join(drop[-1]['tags'])}") + drop = drop[:-1] + if drop: + versions[a] = drop + rc_tags = sorted( + ( + t + for t in rm.git(repo, "tag", "-l", f"{PREFIX}v{x}.{y}.*").split() + if rc_re.fullmatch(t) + ), + key=_vkey, + ) + return { + "ga": ga, + "used": used, + "rc_tags": rc_tags, + "versions": versions, + "kept": kept, + } + + +def promote_blockers(state, branch, target): + """Older lines not GA for `target` yet, one refusal line each: Promote of `branch` for `target` waits for them. + + An older line blocks only where `target` is enabled on it and it has no GA for `target`; GA is the cloud + promote retag or the on-prem chart push. A retired line is gone from the state, so it never blocks. + """ + # Not applied to a KEPT target (OSS). The order protects customers who stay on a line and upgrade later + # (on-prem windows). OSS community users run only :latest, which moves only on the newest line, so an older + # line shipping after a newer one reaches nobody. Each OSS line goes GA with its cloud line, so the order + # would hold a community release on the newest line until the previous cloud line is GA. + if target in KEPT: + return [] + line = _key(branch.split("/", 1)[1]) + return [ + f"{b['branch']} is not GA for {target} yet: promote and publish it first, retire it, or force with a reason" + for b in state["branches"] + if _key(b["line"]) < line + and target in b["targets"] + and not b["targets"][target]["ga"] + ] + + +def enable_blockers(state, branch, target): + """Newer lines already GA for `target`, one refusal line each: `target` may not be enabled on `branch`. + + A target is enabled only on a line newer than every line already GA for it, so an older line can never + wait to go GA behind a newer GA line whose patches the promote order would then hold. A target already + enabled on `branch` is not affected (a patch RC on an old line still builds). + """ + row = next((b for b in state["branches"] if b["branch"] == branch), None) + if row and target in row["targets"]: + return [] + line = _key(branch.split("/", 1)[1]) + return [ + f"{branch} is older than {b['branch']}, already GA for {target}: enable {target} on a newer line instead" + for b in state["branches"] + if _key(b["line"]) > line + and target in b["targets"] + and b["targets"][target]["ga"] + ] + + +def summary(state): + """Markdown table of derive()'s output for $GITHUB_STEP_SUMMARY; the JSON stays the machine input.""" + + def cell(tt): + if not tt: + return "-" + if tt["state"] == "stabilized": + return f"stabilized (GA {tt['ga']['tag']})" + if tt["state"] == "stabilizing" and tt["rcs"]: + return f"stabilizing (rc.{VER_RE.fullmatch(tt['rcs'][-1])[4]})" + return tt["state"] + + out = [ + f"| branch | state | {' | '.join(TARGETS)} | tip_rc | next_rc | ahead_of_ga |", + "|---|---|" + "---|" * (len(TARGETS) + 3), + ] + out += [ + f"| {b['branch']} | {b['state']} | {' | '.join(cell(b['targets'].get(t)) for t in TARGETS)} " + f"| {b['tip_rc'] or '-'}{f' ([Slack]({u}))' if (u := b.get('tip_rc_slack')) else ''} | {b['next_rc'] or '-'} | {'-' if b['ahead_of_ga'] is None else b['ahead_of_ga']} |" + for b in state["branches"] + ] + return ( + "\n".join( + out + + [ + "", + "- supported: " + + "; ".join(f"{t} {v}" for t, v in state["supported"].items()), + f"- backport_targets.fix: {state['backport_targets']['fix']}", + f"- retire: {state['retire']}", + f"- next_minor: {state['next_minor']}", + f"- anomalies: {len(state['anomalies'])}", + ] + ) + + "\n" + ) + + +def main(argv=None, registry=registry_tags): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--repo", default=".") + p.add_argument("--now", type=_ts, default=None, help="ISO-8601 UTC; default: now") + p.add_argument("--rc", help="also report the stable tag promoting this RC would get") + a = p.parse_args(argv) + try: + state = derive(a.repo, a.now or dt.datetime.now(dt.UTC), registry, a.rc) + except ( + Exception + ) as e: # registry or git failure: no JSON, never a silently empty state + print(f"relstate: {e}", file=sys.stderr) + return 1 + print(json.dumps(state, indent=2)) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"state={json.dumps(state, separators=(',', ':'))}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release/ruff.toml b/.github/release/ruff.toml new file mode 100644 index 0000000000..b829599339 --- /dev/null +++ b/.github/release/ruff.toml @@ -0,0 +1,7 @@ +# The repo's ruff settings, plus the release tooling's own style: short local lambdas, `l` for a label, and tests +# that set GITHUB_REPOSITORY before importing relstate. +extend = "../../pyproject.toml" + +[lint.extend-per-file-ignores] +"*.py" = ["E731", "E741"] +"test_*.py" = ["E402"] diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py new file mode 100644 index 0000000000..dfd93f8c4c --- /dev/null +++ b/.github/release/test_backport.py @@ -0,0 +1,927 @@ +"""backport helpers: held_filter and the held summary line; PR body of a pick PR; picks in main order; branches filter; accept= and UNCLASSIFIABLE; owed-picks summary; dispatch route for a non-fix; push summary (status rows, latest tag, stale mark, quiet line); push rescan plan; PR listing classes; push retry; stuck clean pick PR; open conflict PRs per line. Run: python test_backport.py""" + +import io +import json +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +os.environ["GITHUB_REPOSITORY"] = ( + "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets +) +import backport as bp +import relmgr as rm + + +class Held(unittest.TestCase): + def test_held_filter(self): + a, b, c, d, e = (ch * 40 for ch in "abcde") + classes = { + a: ("security", "#1"), + b: ("critical", "#2"), + c: ("fix", "#3"), + d: (None, "no PR"), + e: ("security", "#5"), + } + # a picked (abbreviated trailer), e declined (same trailer): only b is held; fix and non-fix never are. + self.assertEqual( + bp.held_filter([a, b, c, d, e], classes, {a[:7], e}), [(b, "critical", "#2")] + ) + # accept=OWED adds plain fixes; UNCLASSIFIABLE (no PR, bad title) is never dropped, held or owed. + classes[d] = (bp.UNCLASSIFIABLE, "no PR") + self.assertEqual( + bp.held_filter([a, b, c, d, e], classes, {a[:7], e}), + [(b, "critical", "#2"), (d, bp.UNCLASSIFIABLE, "no PR")], + ) + self.assertEqual( + bp.held_filter([a, b, c, d, e], classes, {a[:7], e}, accept=bp.OWED), + [(b, "critical", "#2"), (c, "fix", "#3"), (d, bp.UNCLASSIFIABLE, "no PR")], + ) + rows = [ + (a, "release/0.189", "security", "#1"), + (a, "release/0.182", "security", "#1"), + (b, "release/0.182", "critical", "#2"), + ] + self.assertEqual( + bp.held_warnings(rows), + [ + "::warning::release/0.189: 1 held security/critical fix, Backport with confirm_stabilized", + "::warning::release/0.182: 2 held security/critical fixes, Backport with confirm_stabilized", + ], + ) + self.assertEqual( + bp.held_line(b, "critical", "#2", "release/0.182"), + "::warning::bbbbbbb (critical, PR #2) held for release/0.182: stabilized, needs confirm_stabilized", + ) + + self.assertIsNone(bp.held_note([], "release/0.182")) + with mock.patch.dict( + os.environ, {"GITHUB_SERVER_URL": "", "GITHUB_REPOSITORY": ""} + ): + self.assertEqual( + bp.held_note( + [(b, "critical", "#2", "t"), (d, bp.UNCLASSIFIABLE, "no PR", "t")], + "release/0.182", + ), + f"⚠ Held for the RM on `release/0.182` (stabilized; Promote blocks unless forced): #2 critical, no PR " + f"{bp.UNCLASSIFIABLE}. Pick with `gh workflow run backport.yaml -f commits=#2,{d[:12]} " + "-f branches=release/0.182 -f confirm_stabilized=true`", + ) + + +class Rescan(unittest.TestCase): + """Push mode over held_filter output for nested ranges: 0.190 (stabilizing, cut after a) inside 0.187.""" + + def test_plan(self): + a, b, c, d, e, u = (ch * 40 for ch in "abcdeu") + classes = { + a: ("fix", "#1", "A"), + b: ("fix", "#2", "B"), + c: ("security", "#3", "C"), + d: ("fix", "#4", "D"), + e: ("fix", "#5", "E"), + u: (bp.UNCLASSIFIABLE, "no PR", "U"), + } + targets = { + "fix": ["release/0.190"], + "security_critical": [ + {"branch": "release/0.190", "needs_rm_confirmation": False}, + {"branch": "release/0.187", "needs_rm_confirmation": True}, + ], + } + scan, others = bp.rescan_targets(targets, []) + self.assertEqual( + (scan, others), ({"release/0.190": bp.OWED, "release/0.187": bp.HELD}, []) + ) + # 0.190: b picked, e declined (a decline carries the pick trailer). 0.187 (stabilized): nothing yet. + held = { + "release/0.190": bp.held_filter( + [b, c, d, e, u], classes, {b[:7], e}, scan["release/0.190"] + ), + "release/0.187": bp.held_filter( + [a, b, c, d, e, u], classes, set(), scan["release/0.187"] + ), + } + plan, awaiting, unclassifiable = bp.rescan_plan(targets, held, confirm=False) + # (1) d is missing on 0.190 only: planned there; b, already picked on 0.190, is not. + # (2) declined e is planned nowhere. (3) main order (c before d). (4) security c on stabilized 0.187 waits. + self.assertEqual( + plan, + [ + ("release/0.190", c, "security", "#3", "C"), + ("release/0.190", d, "fix", "#4", "D"), + ], + ) + self.assertEqual(awaiting, [(c, "release/0.187", "security", "#3")]) + self.assertEqual( + unclassifiable, + [(u, "release/0.190", "no PR", "U"), (u, "release/0.187", "no PR", "U")], + ) + plan, awaiting, _ = bp.rescan_plan(targets, held, confirm=True) + self.assertEqual( + (plan[-1], awaiting), (("release/0.187", c, "security", "#3", "C"), []) + ) + + def test_branches_narrow(self): + targets = { + "fix": ["release/0.190"], + "security_critical": [ + {"branch": "release/0.187", "needs_rm_confirmation": True} + ], + } + self.assertEqual( + bp.rescan_targets(targets, ["release/0.187", "release/0.1"]), + ({"release/0.187": bp.HELD}, ["release/0.1"]), + ) + + +class Listing(unittest.TestCase): + def test_classes_and_fallback(self): + with tempfile.TemporaryDirectory() as repo: + git = lambda *a: rm.git( + repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a + ) + git("init", "-q", "-b", "main") + shas = [] + for i in range(6): # cut, fix, security, misc, bad title, no PR + git("commit", "-q", "--allow-empty", "-m", f"c{i}") + shas.append(git("rev-parse", "HEAD")) + cut, fix, sec, misc, bad, nopr = shas + git("update-ref", "refs/remotes/origin/main", nopr) + git("update-ref", "refs/remotes/origin/release/0.1", cut) + pr = lambda n, sha, title, labels=(): { + "number": n, + "title": title, + "labels": [{"name": l} for l in labels], + "mergeCommit": {"oid": sha}, + } + listing = [ + pr(1, fix, "UN-1 [FIX] a"), + pr(2, sec, "UN-2 [FIX] b", ["fix/security"]), + pr(3, misc, "UN-3 [MISC] c"), + pr(4, bad, "no prefix"), + {"number": 5, "title": "open", "labels": [], "mergeCommit": None}, + ] + classes = bp.classes_from_listing(listing) + self.assertEqual(classes[fix], ("fix", "#1", "UN-1 [FIX] a")) + self.assertEqual(classes[sec], ("security", "#2", "UN-2 [FIX] b")) + self.assertEqual(classes[misc], (None, "#3", "UN-3 [MISC] c")) + self.assertEqual(classes[bad][0], bp.UNCLASSIFIABLE) + self.assertIn( + "no recognised '[TYPE]' or 'TICKET [TYPE]' prefix", classes[bad][1] + ) + self.assertEqual( + len(classes), 4 + ) # an unmerged row (no merge commit) is skipped + looked = [] + saved = bp.fix_class + bp.fix_class = lambda slug, sha, token=None: looked.append(sha) or ( + None, + "no PR", + None, + ) + try: + rows = bp.held("o/r", repo, "release/0.1", accept=bp.OWED, cache=classes) + finally: + bp.fix_class = saved + self.assertEqual( + looked, [nopr] + ) # only the commit the listing missed is looked up + self.assertEqual( + [(s, c) for s, c, *_ in rows], + [ + (fix, "fix"), + (sec, "security"), + (bad, bp.UNCLASSIFIABLE), + (nopr, bp.UNCLASSIFIABLE), + ], + ) + + +class PushRetry(unittest.TestCase): + def test_retry_then_fail_loudly(self): + sha, head, branch = "c" * 40, "backport/0.190/ccccccc", "release/0.190" + rejected = "To https://github.com/o/r\n ! [remote rejected] HEAD -> backport/0.190/ccccccc (failure)" + calls, sleeps = [], [] + + def fake(results): + def run(*args, check=True, input=None): + calls.append(args) + rc = results.pop(0) if args[:2] == ("git", "push") else 0 + return subprocess.CompletedProcess(args, rc, "", rejected if rc else "") + + return run + + saved, env = bp.run, dict(os.environ) + os.environ.update( + GITHUB_SERVER_URL="https://github.com", + GITHUB_REPOSITORY="o/r", + GITHUB_RUN_ID="9", + ) + try: + bp.run = fake( + [1, 1, 0] + ) # rejected twice, then pushed: one success, two waits, no comment + self.assertIsNone( + bp.push_head("o/r", sha, branch, head, "#7", sleep=sleeps.append) + ) + self.assertEqual((sleeps, len(calls)), ([5, 20], 3)) + calls.clear(), sleeps.clear() + bp.run = fake([1, 1, 1]) + with mock.patch.object(bp.rm, "is_converged", lambda *a, **k: False): + out = bp.push_head("o/r", sha, branch, head, "#7", sleep=sleeps.append) + comment = calls[-1] + calls.clear(), sleeps.clear() + bp.run = fake( + [1, 1, 1] + ) # rejected because another run landed the pick: no failure, no comment + with mock.patch.object(bp.rm, "is_converged", lambda *a, **k: True): + landed = bp.push_head("o/r", sha, branch, head, "#7", sleep=sleeps.append) + finally: + bp.run = saved + os.environ.clear(), os.environ.update(env) + self.assertEqual( + out, + ( + "failed", + f"push of {head} rejected 3 times: ! [remote rejected] HEAD -> {head} (failure)", + ), + ) + self.assertEqual(landed[0], "skipped") + self.assertEqual(sleeps, [5, 20]) + self.assertNotIn(("gh", "pr"), [a[:2] for a in calls]) + self.assertEqual(comment[:6], ("gh", "pr", "comment", "7", "-R", "o/r")) + self.assertEqual( + comment[-1].splitlines()[0], + f"Backport to {branch} failed: push of {head} rejected (run https://github.com/o/r/actions/runs/9)." + f" Rerun Backport for ccccccc on {branch}.", + ) + + +ORIG = "77212a38cc07a0e7fc5f31c691275666976c4c82" + + +class Body(unittest.TestCase): + def test_conflict_body_has_no_stale_rule(self): + body = bp.pr_body( + ORIG, "#57", "release/0.185", "backport/0.185/77212a3", ["NOTES.md"] + ) + self.assertNotIn("do not merge by hand", body) + self.assertIn( + "Resolve, get one approval and merge, or close the PR to decline (the close comment is the reason).", + body, + ) + self.assertNotIn("Declined:", body) + self.assertNotIn("backport-approved", body) + self.assertIn("- `NOTES.md`", body) + self.assertIn( + f"push a fix-up commit; the first commit's `(cherry picked from commit {ORIG})` line carries", + body, + ) + self.assertIn( + "Clean pick", + bp.pr_body(ORIG, "#57", "release/0.185", "backport/0.185/77212a3"), + ) + + +class MainOrder(unittest.TestCase): + def test_newest_first_comes_back_oldest_first(self): + with tempfile.TemporaryDirectory() as repo: + rm.git(repo, "init", "-q", "-b", "main") + shas = [] + for i, date in enumerate(("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z")): + os.environ["GIT_COMMITTER_DATE"] = date + try: + rm.git( + repo, + "-c", + "user.name=Dev", + "-c", + "user.email=dev@example.com", + "commit", + "-q", + "--allow-empty", + "-m", + f"fix {i}", + ) + finally: + del os.environ["GIT_COMMITTER_DATE"] + shas.append(rm.git(repo, "rev-parse", "HEAD")) + self.assertEqual(bp.main_order(repo, [shas[1], shas[0], shas[1]]), shas) + + +class OwedSummary(unittest.TestCase): + def test_table_and_backport_line(self): + a, b = "a" * 40, "b" * 40 + out = bp.owed_summary( + [(a, "fix", "#3", "UN-1 [FIX] x | y"), (b, bp.UNCLASSIFIABLE, "no PR", "z")], + "release/0.187", + ) + self.assertIn("| `aaaaaaa` | #3 | UN-1 [FIX] x \\| y | fix |", out) + self.assertIn(f"| `bbbbbbb` | no PR | z | {bp.UNCLASSIFIABLE} |", out) + self.assertTrue( + out.endswith(f"Backport commits: {a}") + ) # UNCLASSIFIABLE stays out of the paste line + self.assertTrue(bp.owed_summary([], "release/0.187").endswith("none")) + + +class OnlyListed(unittest.TestCase): + def test_filter_narrows_never_adds(self): + targets = ["release/0.189", "release/0.188"] + self.assertEqual(bp.only_listed(targets, []), (targets, [])) + self.assertEqual( + bp.only_listed(targets, ["release/0.188"]), (["release/0.188"], []) + ) + self.assertEqual( + bp.only_listed(["release/0.189"], ["release/0.188", "release/0.189"]), + (["release/0.189"], ["release/0.188"]), + ) + + +class DispatchNonFix(unittest.TestCase): + def test_row_names_the_route(self): + a, b = "a" * 40, "b" * 40 + classes = {a: (None, "#250", "UN-1 [MISC] docs"), b: (None, "no PR", None)} + with ( + mock.patch.object(bp, "resolve", lambda slug, c: c), + mock.patch.object(bp, "main_order", lambda repo, shas: shas), + mock.patch.object(bp, "fix_class", lambda slug, sha: classes[sha]), + ): + rows, awaiting = bp.dispatch( + "o/r", + {"fix": ["release/0.191"], "security_critical": []}, + f"{a} {b}", + False, + [], + ) + self.assertEqual( + rows, + [ + ( + a, + "#250", + "none", + "-", + "skipped: not a fix; label #250 fix/needs-backport (or the fix label it needs) and rerun", + "-", + ), + (b, "no PR", "none", "-", "skipped: not a fix", "-"), + ], + ) + self.assertEqual(awaiting, []) + + +class PushSummary(unittest.TestCase): + def setUp(self): # plain-text links, whatever the shell exports + self.env = mock.patch.dict( + os.environ, {"GITHUB_SERVER_URL": "", "GITHUB_REPOSITORY": ""} + ) + self.env.start() + self.addCleanup(self.env.stop) + + def test_next_step_first_match_wins(self): + row = lambda **k: bp.status_row( + "release/0.192", + "stabilizing", + **{ + "tag": "v0.192.0-rc.2", + "unbuilt": 0, + "missing": 0, + "conflicts": [], + "held_n": 0, + **k, + }, + ) + self.assertEqual( + row(conflicts=[243], held_n=4, unbuilt=1), + "| `release/0.192` | stabilizing | `v0.192.0-rc.2` · all built | 0 | 1 conflict, 4 RM | resolve #243 |".replace( + "all built", "+1 not built" + ), + ) + self.assertTrue( + row(held_n=4, unbuilt=1).endswith("| 4 RM | RM: confirm 4 security fixes |") + ) + self.assertTrue( + row(held_n=1, unbuilt=1).endswith("| 1 RM | RM: confirm 1 security fix |") + ) + self.assertTrue(row(unbuilt=1).endswith("+1 not built | 0 | - | Take RC |")) + self.assertTrue(row().endswith("all built | 0 | - | Promote |")) + self.assertTrue( + row(tag="v0.192.0").endswith("`v0.192.0` · all built | 0 | - | - |") + ) + self.assertIn("| none |", row(tag=None, unbuilt=None)) + + def test_latest_tag_counts_since_any_tag(self): + with tempfile.TemporaryDirectory() as repo: + rm.git(repo, "init", "-q", "-b", "release/0.1") + commit = lambda m: rm.git( + repo, + "-c", + "user.name=Dev", + "-c", + "user.email=dev@example.com", + "commit", + "-q", + "--allow-empty", + "-m", + m, + ) + commit("cut") + rm.git(repo, "tag", "v0.1.0-rc.1") + rm.git(repo, "tag", "v0.1.0") # promoted: same commit, the stable tag wins + rm.git( + repo, "tag", "v0.10.0" + ) # another line's tag on the same commit is not this line's + rm.git(repo, "update-ref", "refs/remotes/origin/release/0.1", "HEAD") + self.assertEqual(bp.latest_tag(repo, "release/0.1"), ("v0.1.0", 0)) + commit("fix") + rm.git(repo, "tag", "v0.1.1-rc.1") + commit("fix 2") + rm.git(repo, "update-ref", "refs/remotes/origin/release/0.1", "HEAD") + self.assertEqual( + bp.latest_tag(repo, "release/0.1"), ("v0.1.1-rc.1", 1) + ) # since the RC, not since v0.1.0 + + def test_stale_mark_from_five_days(self): + self.assertEqual(bp.stale(4), "4 days") + self.assertEqual(bp.stale(5), "5 days ⚠") + self.assertEqual(bp.stale(1), "1 day") + + def test_quiet_push_is_one_line(self): + self.assertEqual( + bp.push_summary( + "Backport · push `abc1234` (#250, not a fix)", ["| row |"], [], [] + ), + [ + "Backport · push `abc1234` (#250, not a fix) · nothing picked, nothing pending" + ], + ) + out = bp.push_summary( + "Backport · push `abc1234` (#251, fix)", ["| row |"], ["- done"], [] + ) + self.assertEqual(out[0], "### Backport · push `abc1234` (#251, fix)") + self.assertIn("**This run**", out) + out = bp.push_summary( + "Backport · push `abc1234` (#252, not a fix)", ["| row |"], [], ["
"] + ) + self.assertIn( + "**This run:** nothing picked", out + ) # something pending: never the quiet line + + def test_this_run_and_pending(self): + a = "a" * 40 + rows = [ + (a, "#251", "fix", "release/0.192", "merged 1234567 (#260)", "-"), + ( + a, + "#251", + "fix", + "release/0.190", + "pr https://github.com/o/r/pull/261", + "-", + ), + (a, "#251", "fix", "release/0.189", "skipped: already on branch", "-"), + ] + self.assertEqual( + bp.this_run(rows), + [ + "- #251 → `release/0.192`: merged as `1234567` (#260)", + "- #251 → `release/0.190`: conflict, #261 to resolve", + ], + ) + out = bp.pending_block( + [(261, "release/0.190", 6)], + [ + (a, "release/0.190", "security", "#247", 2), + (a, "release/0.189", "security", "#247", 2), + ], + ) + self.assertEqual(out[0], "
Pending (2)") + self.assertIn("- #261 on `release/0.190` · open 6 days ⚠", out) + self.assertIn( + "- #247 security → `release/0.190`, `release/0.189` · merged 2 days ago", out + ) + self.assertIn( + "gh workflow run backport.yaml -f commits=#247 -f branches=release/0.190,release/0.189 " + "-f confirm_stabilized=true", + out, + ) + self.assertEqual(bp.pending_block([], []), []) + + +class ConflictPRs(unittest.TestCase): + """Open backport-conflict PRs of one repo, per line, read by a stub gh as the given token; a failed listing raises.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.d = tmp.name + with open(os.path.join(self.d, "gh"), "w") as fh: + fh.write( + f'#!/bin/sh\necho "$GH_TOKEN $*" >> {self.d}/gh.log\n[ -e {self.d}/fail ] && exit 1\ncat {self.d}/prs.json\n' + ) + os.chmod(os.path.join(self.d, "gh"), 0o755) + p = mock.patch.dict( + os.environ, {"PATH": f"{self.d}:{os.environ['PATH']}", "GH_TOKEN": "cloud"} + ) + p.start() + self.addCleanup(p.stop) + pr = lambda n, base: { + "number": n, + "url": f"https://x/{n}", + "baseRefName": base, + "createdAt": "2026-10-01T00:00:00Z", + } + with open(os.path.join(self.d, "prs.json"), "w") as fh: + json.dump([pr(7, "release/1.7"), pr(8, "release/1.6")], fh) + + def log(self): + with open(os.path.join(self.d, "gh.log")) as fh: + return fh.read() + + def test_lists_one_line_as_the_token(self): + self.assertEqual( + [p["number"] for p in bp.conflict_prs("o/oss", "release/1.7", "oss")], [7] + ) + self.assertEqual([p["number"] for p in bp.conflict_prs("o/r")], [7, 8]) + log = self.log().splitlines() + self.assertTrue( + log[0].startswith( + "oss pr list -R o/oss --label backport-conflict --state open" + ), + log[0], + ) + self.assertTrue(log[1].startswith("cloud pr list -R o/r "), log[1]) + + def test_open_conflicts(self): + with mock.patch("sys.stdout", new_callable=io.StringIO) as out: + refs, note = bp.open_conflicts( + [("o/r", "release/1.6", None), ("o/oss", "release/1.7", "oss")], "error" + ) + self.assertEqual(refs, ["o/r#8", "o/oss#7"]) + self.assertIn( + "::error::Open backport conflict PR o/oss#7 on release/1.7", out.getvalue() + ) + self.assertEqual( + note, + "⚠ Open backport conflict PRs: [o/r#8](https://x/8) on `release/1.6`, " + "[o/oss#7](https://x/7) on `release/1.7`", + ) + with mock.patch("sys.stdout", new_callable=io.StringIO): + self.assertEqual( + bp.open_conflicts([("o/r", "release/1.9", None)]), ([], None) + ) + + def test_failed_listing_raises(self): + open(os.path.join(self.d, "fail"), "w").close() + with self.assertRaises(RuntimeError): + bp.open_conflicts([("o/r", "release/1.7", None)]) + + +class FinishOpen(unittest.TestCase): + """A pick head left by an earlier run: an unlabelled one-commit PR is merged now, anything else waits.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + d = self.tmp.name + g = ( + lambda *a, cwd=d + "/w": subprocess.run( + ["git", "-C", cwd, *a], check=True, capture_output=True, text=True + ).stdout.strip() + ) + subprocess.run( + ["git", "init", "-q", "--bare", "-b", "main", d + "/o"], check=True + ) + subprocess.run( + ["git", "clone", "-q", d + "/o", d + "/w"], check=True, capture_output=True + ) + g( + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "base", + ) + g("push", "-q", "origin", "HEAD:refs/heads/release/1.0") + g( + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "fix: x\n\n(cherry picked from commit " + "a" * 40 + ")", + ) + g("push", "-q", "origin", "HEAD:refs/heads/backport/1.0/aaaaaaa") + g("fetch", "-q", "origin") + self.oid, self.g, self.merged = g("rev-parse", "HEAD"), g, [] + self.cwd = os.getcwd() + os.chdir(d + "/w") + + def tearDown(self): + os.chdir(self.cwd) + self.tmp.cleanup() + + def finish(self, labels=(), oid=None, prs=None): + listing = [ + { + "number": 4, + "headRefOid": oid or self.oid, + "labels": [{"name": x} for x in labels], + } + ] + squash = lambda slug, pr, msg: self.merged.append((pr["number"], msg)) or True + with ( + mock.patch.object( + bp, "gh", lambda *a: json.dumps(listing if prs is None else prs) + ), + mock.patch.object(bp.bm, "squash", squash), + mock.patch.object(bp, "soft", lambda *a: None), + ): + return bp.finish_open("o/r", "release/1.0", "backport/1.0/aaaaaaa") + + def push( + self, msg, path=None + ): # replace the head's one commit with one carrying msg (and a file at path) + self.g("reset", "-q", "--hard", "origin/release/1.0") + if path: + with open(path, "w") as f: + f.write("<<<<<<< HEAD\na\n=======\nb\n>>>>>>> fix\n") + self.g("add", path) + self.g( + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + msg, + ) + self.g("push", "-q", "-f", "origin", "HEAD:refs/heads/backport/1.0/aaaaaaa") + return self.g("rev-parse", "HEAD") + + def test_clean_pick_merged_with_its_trailer(self): + out = self.finish() + self.assertEqual(out[0], "merged") + self.assertEqual(self.merged[0][0], 4) + self.assertIn("(cherry picked from commit " + "a" * 40 + ")", self.merged[0][1]) + + def test_waits_for_a_person(self): + pending = ("skipped", "conflict PR pending on `backport/1.0/aaaaaaa`") + self.assertEqual( + self.finish(labels=["backport-conflict"]), pending + ) # conflict PR + self.g( + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "person", + ) + self.g("push", "-q", "origin", "HEAD:refs/heads/backport/1.0/aaaaaaa") + self.assertEqual( + self.finish(oid=self.g("rev-parse", "HEAD")), pending + ) # a person's commit on top + self.assertEqual(self.merged, []) + + def test_head_without_a_pr_fails(self): + out = self.finish( + prs=[] + ) # pushed just now: the run that pushed it has yet to open its PR + self.assertEqual( + out, + ( + "skipped", + "head backport/1.0/aaaaaaa is 0 min old with no open PR: another run is picking it", + ), + ) + with mock.patch.object(bp, "PICK_TIMEOUT", 0): + out = self.finish(prs=[]) + self.assertEqual( + out, + ( + "failed", + "head backport/1.0/aaaaaaa exists with no open PR: delete it " + "or reopen its PR, then rerun", + ), + ) + + def test_unlabelled_conflict_pick_is_not_squashed(self): + trailer = "(cherry picked from commit " + "a" * 40 + ")" + for oid in ( + self.push(f"fix: x\n\n{trailer}", path="f.txt"), # markers in the tree + self.push(f"fix: x\n\n{trailer}\n\nConflicts:\n\tf.txt\n"), + ): # the block pick writes + out = self.finish(oid=oid) + self.assertEqual(out[0], "failed") + self.assertIn("without the backport-conflict label", out[1]) + self.assertEqual(self.merged, []) + + +class Pick(unittest.TestCase): + """pick() against a local origin: an empty pick names its fix and the decline; fix/* labels reach the pick PR.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + d = tmp.name + self.g = g = lambda *a: subprocess.run( + ["git", "-C", d + "/w", "-c", "user.name=t", "-c", "user.email=t@t", *a], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + subprocess.run( + ["git", "init", "-q", "--bare", "-b", "main", d + "/o"], check=True + ) + subprocess.run( + ["git", "clone", "-q", d + "/o", d + "/w"], check=True, capture_output=True + ) + self.write(d + "/w/a.txt", "x\n") + ( + g("add", "a.txt"), + g("commit", "-q", "-m", "base"), + g("push", "-q", "origin", "HEAD:main", "HEAD:release/1.0"), + ) + self.write(d + "/w/a.txt", "y\n") + g("commit", "-q", "-am", "fix: y (#5)"), g("push", "-q", "origin", "HEAD:main") + self.fix = g("rev-parse", "HEAD") + g("fetch", "-q", "origin") + self.d, cwd = d, os.getcwd() + os.chdir(d + "/w") + self.addCleanup(os.chdir, cwd) + self.calls = [] + + @staticmethod + def write(path, text): + with open(path, "w") as f: + f.write(text) + + def gh(self, *a, token=None): + self.calls.append(a) + if a[:2] == ("api", "repos/o/r/pulls/5"): + return "fix/security\nfix/needs-backport\nother" + return "https://github.com/o/r/pull/8" + + def pick(self): + with ( + mock.patch.object(bp, "gh", self.gh), + mock.patch.object(bp.bm, "squash", lambda *a: True), + mock.patch.object(bp, "soft", lambda *a: None), + ): + return bp.pick("o/r", self.fix, "release/1.0", "UN-1 [FIX] y", "#5") + + def test_clean_pick_carries_fix_labels(self): + self.assertEqual(self.pick()[0], "merged") + create = next(a for a in self.calls if a[:2] == ("pr", "create")) + self.assertEqual( + [create[i + 1] for i, x in enumerate(create) if x == "--label"], + ["fix/security", "fix/needs-backport"], + ) + + def test_conflict_pick_carries_fix_labels(self): + self.g("checkout", "-q", "-B", "rel", "origin/release/1.0") + self.write(self.d + "/w/a.txt", "z\n") + ( + self.g("commit", "-q", "-am", "z"), + self.g("push", "-q", "origin", "HEAD:release/1.0"), + self.g("fetch", "-q", "origin"), + ) + self.assertEqual(self.pick()[0], "pr") + create = next(a for a in self.calls if a[:2] == ("pr", "create")) + self.assertEqual( + [create[i + 1] for i, x in enumerate(create) if x == "--label"], + ["backport-conflict", "fix/security", "fix/needs-backport"], + ) + + def test_empty_pick_names_the_decline(self): + self.g("checkout", "-q", "-B", "rel", "origin/release/1.0") + self.write(self.d + "/w/a.txt", "y\n") + self.write(self.d + "/w/b.txt", "b\n") + self.g("add", "."), self.g("commit", "-q", "-m", "y and b, by hand") + ( + self.g("push", "-q", "origin", "HEAD:release/1.0"), + self.g("fetch", "-q", "origin"), + ) + out = self.pick() + self.assertEqual(out[0], "failed") + self.assertIn(f"pick of {self.fix[:7]} onto release/1.0 is empty", out[1]) + self.assertIn( + f"`(cherry picked from commit {self.fix})` and `Declined: already on release/1.0`", + out[1], + ) + self.assertNotIn(("pr", "create"), [a[:2] for a in self.calls]) + + def test_silent_failure_names_the_exit(self): + real = bp.run + fake = lambda *a, **k: ( + subprocess.CompletedProcess(a, 3, "", "") + if a[:3] == ("git", "cherry-pick", "-x") + else real(*a, **k) + ) + with mock.patch.object(bp, "run", fake): + self.assertEqual(self.pick(), ("failed", "cherry-pick exit 3")) + + +class PickRow(unittest.TestCase): + def test_relevance_failure_never_fails_the_pick(self): + with ( + mock.patch.object( + bp.rm, "relevance", side_effect=RuntimeError("bad | path\nsecond line") + ), + mock.patch.object(bp, "pick", lambda *a: ("merged", "abc1234 (#8)")), + mock.patch("sys.stdout", io.StringIO()), + ): + row = bp.pick_row("o/r", "c" * 40, "release/1.0", "fix", "#5", "t") + self.assertEqual(row[4], "merged abc1234 (#8)") + self.assertEqual(row[5], "unknown (RuntimeError: bad \\| path)") + + def test_failure_names_its_type(self): + with ( + mock.patch.object(bp.rm, "relevance", lambda *a: "relevant"), + mock.patch.object(bp, "pick", side_effect=KeyError("x")), + mock.patch.object(bp, "run"), + mock.patch("sys.stdout", io.StringIO()), + mock.patch("sys.stderr", io.StringIO()) as err, + ): + row = bp.pick_row("o/r", "c" * 40, "release/1.0", "fix", "#5", "t") + self.assertEqual(row[4], "failed: KeyError: 'x'") + self.assertIn("Traceback", err.getvalue()) + + +class PushTitle(unittest.TestCase): + def test_unclassifiable_push_still_summarised(self): + with ( + mock.patch.object(bp, "fix_class", side_effect=ValueError("no [TYPE]")), + mock.patch("sys.stdout", new_callable=io.StringIO) as out, + ): + self.assertTrue( + bp.push_title("o/r", "c" * 40).endswith(f" (-, {bp.UNCLASSIFIABLE})") + ) + self.assertIn("::warning::ccccccc: fix class: no [TYPE]", out.getvalue()) + + +class Soft(unittest.TestCase): + def test_failure_warns(self): + with mock.patch("sys.stdout", new_callable=io.StringIO) as out: + bp.soft("delete of h", "sh", "-c", "echo gone >&2; exit 3") + bp.soft("delete of h", "true") + self.assertEqual(out.getvalue(), "::warning::delete of h failed (exit 3): gone\n") + + +class Relevance(unittest.TestCase): + """relmgr.relevance: are the line versions a main fix edits on the branch?""" + + def test_cases(self): + with tempfile.TemporaryDirectory() as d: + g = lambda *a: subprocess.run( + ["git", "-C", d, "-c", "user.name=t", "-c", "user.email=t@t", *a], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def commit(msg, **files): + for name, text in files.items(): + with open(os.path.join(d, name.replace("_", " ")), "w") as f: + f.write(text) + g("add", "-A"), g("commit", "-q", "-m", msg) + return g("rev-parse", "HEAD") + + g("init", "-q", "-b", "main") + commit("base", a="1\n2\n", b="1\n", c_d="1\n") + g("branch", "release/1.0") + main = commit("rewrite a:2 after the cut", a="1\nTWO\n", b="1\nnew\n") + rel = lambda **files: g( + "checkout", "-q", "-B", "fix", main + ) or bp.rm.relevance(d, commit("fix", **files), "release/1.0") + self.assertEqual(rel(a="ONE\nTWO\n"), "relevant") # a:1 is as on the branch + self.assertEqual( + rel(a="ONE\nTWO!\n"), "partial" + ) # a:1 from the cut, a:2 from after it + self.assertEqual( + rel(b="1\nnewer\n"), "not-relevant" + ) # b:2 exists only after the cut + self.assertEqual(rel(e="added\n"), "unknown") # a new file: no pre-image + self.assertEqual( + rel(c_d="1\nadded\n"), "unknown" + ) # lines added to an existing file + self.assertEqual(rel(c_d="one\n"), "relevant") # a path with a space + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py new file mode 100644 index 0000000000..59eb9d6963 --- /dev/null +++ b/.github/release/test_backport_merge.py @@ -0,0 +1,404 @@ +"""Pure-logic checks for backport_merge (decline reason, landed, merge, close-is-decline, re-runs, reopen, squash retry), with gh +and git injected. Run: python test_backport_merge.py +""" + +import json +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +import backport_merge as bm + +O = "c" * 40 # the main original +CLOSED = "2026-10-12T10:00:00Z" +ME = "closer" +BOT = "release-bot" + + +def comment(body, at=CLOSED, user=ME): + return {"created_at": at, "body": body, "user": user} + + +class Reason(unittest.TestCase): + def test_closers_newest_comment_in_window(self): + cs = [ + comment("Declined: older", "2026-10-12T09:59:00Z"), + comment("Declined: not relevant on 0.185\nmore text", "2026-10-12T09:59:30Z"), + comment("someone else's", "2026-10-12T09:59:50Z", user="other"), + ] + self.assertEqual(bm.decline_reason(cs, ME, CLOSED), "not relevant on 0.185") + + def test_prefix_optional(self): + self.assertEqual( + bm.decline_reason([comment("superseded by #12")], ME, CLOSED), + "superseded by #12", + ) + + def test_none(self): + self.assertIsNone(bm.decline_reason([], ME, CLOSED)) + self.assertIsNone(bm.decline_reason([comment("Declined: ")], ME, CLOSED)) + self.assertIsNone( + bm.decline_reason( + [comment("Declined: old", "2026-10-12T09:57:59Z")], ME, CLOSED + ) + ) + self.assertIsNone( + bm.decline_reason( + [comment("Declined: later", "2026-10-12T10:00:01Z")], ME, CLOSED + ) + ) + self.assertIsNone( + bm.decline_reason([comment("Declined: theirs", user="other")], ME, CLOSED) + ) + + +class Landed(unittest.TestCase): + def test_landed(self): + commits = [ + ("1" * 40, "fix: other\n\n(cherry picked from commit " + "d" * 40 + ")"), + ( + "2" * 40, + "fix: x\n\n(cherry picked from commit " + O + ")\n\nDeclined: n/a", + ), + ] + self.assertEqual(bm.landed([O], commits), "2" * 40) + self.assertIsNone(bm.landed([O], commits[:1])) + self.assertIsNone(bm.landed([O], [])) + + +class Gh: + """Stand-in for bm.gh: records each call; serves the PR view, its comments and the decline commit's sha.""" + + def __init__(self, state, labels=(), comments=()): + self.pr = { + "number": 7, + "state": state, + "closedAt": CLOSED, + "baseRefName": "release/0.187", + "headRefName": "backport/0.187/ccccccc", + "headRefOid": "a" * 40, + "labels": [{"name": n} for n in labels], + "isCrossRepository": False, + "author": {"login": BOT}, + } + self.comments, self.calls = list(comments), [] + + def __call__(self, *args, token=None): + self.calls.append(args) + if args[:2] == ("pr", "view"): + return json.dumps(self.pr) + if args[:2] == ("api", "user"): + return BOT + if args[:2] == ("api", "--paginate"): + return "\n".join(json.dumps(c) for c in self.comments) + if args[:4] == ("api", "-X", "POST", "repos/o/r/git/commits"): + return json.dumps({"sha": "e" * 40}) + return "" + + def writes(self): + """Every call after the reads, as (verb, path-or-subcommand).""" + return [ + a[:2] if a[0] == "pr" else a[1:4] + for a in self.calls + if a[:2] not in (("pr", "view"), ("api", "--paginate"), ("api", "user")) + ] + + def said(self): + return [a[-1] for a in self.calls if a[:2] == ("pr", "comment")] + + +RECORDED = ( + "Recorded as a decline on release/0.187 ({}). To undo: reopen this PR and merge it." +) +LABELS = "repos/o/r/issues/7/labels" + + +class Close(unittest.TestCase): + def setUp(self): + saved = { + k: getattr(bm, k) for k in ("gh", "first_pick", "base_picks", "base_tip") + } + self.addCleanup(lambda: [setattr(bm, k, v) for k, v in saved.items()]) + self.on_base = [] # (sha, message) already on the base, newest first + self.originals = [O] + bm.first_pick = lambda pr: ( + f"fix: x (#9)\n\n(cherry picked from commit {O})", + self.originals, + ) + bm.base_picks = lambda base, originals: self.on_base + bm.base_tip = lambda base: ("b" * 40, "t" * 40) + + def close(self, *a, **k): + bm.gh = Gh(*a, **k) + return bm.close("o/r", 7, ME) + + def test_merged(self): + out = self.close("MERGED", labels=["backport-conflict", "backport-declined"]) + self.assertEqual(out, "merged") + self.assertEqual( + bm.gh.writes(), + [ + ("-X", "DELETE", f"{LABELS}/backport-conflict"), + ("-X", "DELETE", f"{LABELS}/backport-declined"), + ("-X", "POST", LABELS), + ("-X", "DELETE", "repos/o/r/git/refs/heads/backport/0.187/ccccccc"), + ], + ) + self.assertEqual(bm.gh.calls[4][-1], "labels[]=backport-merged") + + def test_close_with_comment_declines_and_keeps_head(self): + out = self.close( + "CLOSED", + labels=["backport-conflict", "backport-merged"], + comments=[comment("Declined: not on 0.187")], + ) + self.assertEqual(out, "declined eeeeeee") + post = next( + a + for a in bm.gh.calls + if a[:4] == ("api", "-X", "POST", "repos/o/r/git/commits") + ) + self.assertIn( + f"message=fix: x (#9)\n\n(cherry picked from commit {O})\n\nDeclined: not on 0.187", + post, + ) + self.assertIn("parents[]=" + "b" * 40, post) + self.assertEqual( + bm.gh.writes(), + [ + ("-X", "POST", "repos/o/r/git/commits"), + ("-X", "PATCH", "repos/o/r/git/refs/heads/release/0.187"), + ("-X", "DELETE", f"{LABELS}/backport-conflict"), + ("-X", "DELETE", f"{LABELS}/backport-merged"), + ("-X", "POST", LABELS), + ("pr", "comment"), + ], + ) # no head delete: a reopen needs it + self.assertEqual(bm.gh.said(), [RECORDED.format("eeeeeee")]) + + def test_close_without_comment_default_reason(self): + self.assertEqual( + self.close("CLOSED", labels=["backport-conflict"]), "declined eeeeeee" + ) + post = next( + a + for a in bm.gh.calls + if a[:4] == ("api", "-X", "POST", "repos/o/r/git/commits") + ) + self.assertIn( + f"message=fix: x (#9)\n\n(cherry picked from commit {O})\n\n" + f"Declined: Closed by @{ME} without a reason (PR #7)", + post, + ) + + def test_rerun_writes_nothing(self): + self.on_base = [ + ( + "d" * 40, + f"fix: x\n\n(cherry picked from commit {O})\n\nDeclined: not on 0.187", + ) + ] + out = self.close( + "CLOSED", + labels=["backport-declined"], + comments=[ + comment("Declined: not on 0.187"), + comment(RECORDED.format("ddddddd"), user="bot"), + ], + ) + self.assertEqual(out, "skipped: already on release/0.187 as ddddddd") + self.assertEqual(bm.gh.writes(), []) + + def test_close_again_after_reopen(self): + """Reopened (label backport-conflict), closed again: no second decline commit; label and comment fixed.""" + self.on_base = [ + ("d" * 40, f"fix: x\n\n(cherry picked from commit {O})\n\nDeclined: first") + ] + out = self.close( + "CLOSED", + labels=["backport-conflict"], + comments=[ + comment(RECORDED.format("ddddddd"), "2026-10-12T09:00:00Z", user="bot") + ], + ) + self.assertEqual(out, "skipped: already on release/0.187 as ddddddd") + self.assertEqual( + bm.gh.writes(), + [ + ("-X", "DELETE", f"{LABELS}/backport-conflict"), + ("-X", "POST", LABELS), + ("pr", "comment"), + ], + ) + self.assertEqual(bm.gh.said(), [RECORDED.format("ddddddd")]) + + def test_lost_trailer_fails_loudly(self): + self.originals = [] + with self.assertRaises(RuntimeError): + self.close("CLOSED", labels=["backport-conflict"]) + self.assertEqual(bm.gh.writes(), [("pr", "comment")]) + self.assertTrue(bm.gh.said()[0].startswith("Bot could not record the decline")) + + def test_reopened_sets_conflict_only(self): + self.assertEqual(self.close("OPEN", labels=["backport-declined"]), "reopened") + self.assertEqual( + bm.gh.writes(), + [("-X", "DELETE", f"{LABELS}/backport-declined"), ("-X", "POST", LABELS)], + ) + self.assertEqual(bm.gh.calls[-1][-1], "labels[]=backport-conflict") + + def test_not_a_backport_pr_refused(self): + """A dispatch with a wrong number: no label, comment or commit.""" + for change in ( + {"headRefName": "feat/x"}, + {"baseRefName": "main"}, + {"isCrossRepository": True}, + {"headRefName": "backport/0.186/ccccccc"}, + {"headRefName": "backport/0.187/ccccccc-x"}, + {"headRefName": "backport/0.187/cccccc"}, + {"author": {"login": ME}}, + ): + bm.gh = Gh("CLOSED", labels=["backport-conflict"]) + bm.gh.pr.update(change) + with self.assertRaises(RuntimeError): + bm.close("o/r", 7, ME) + self.assertEqual(bm.gh.writes(), []) + + def test_head_already_gone(self): + def gh(*args, token=None): + if args[:3] == ("api", "-X", "DELETE") and "/git/refs/" in args[3]: + raise RuntimeError( + "gh api -X (exit 1): Reference does not exist (HTTP 422)" + ) + return fake(*args) + + fake = Gh("MERGED", labels=["backport-merged"]) + bm.gh = gh + self.assertEqual(bm.close("o/r", 7, ME), "merged") + + +class FirstPick(unittest.TestCase): + """The decline names only the main commit of the head's sha7, with main's message, never the head's text.""" + + def test_pick_trailer(self): + t = lambda o: f"(cherry picked from commit {o})" + head = "backport/0.187/ccccccc" + self.assertEqual(bm.rm.pick_trailer(head, f"fix: x\n\n{t(O)}"), O) + self.assertIsNone( + bm.rm.pick_trailer(head, f"fix: x\n\n{t(O)}\n{t('d' * 40)}") + ) # a second fix named + self.assertIsNone( + bm.rm.pick_trailer(head, f"fix: x\n\n{t('d' * 40)}") + ) # another fix than the head's + self.assertIsNone(bm.rm.pick_trailer(head, "fix: x")) + self.assertIsNone(bm.rm.pick_trailer("backport/0.187/ccc", f"fix: x\n\n{t(O)}")) + + def test_message_from_main(self): + with tempfile.TemporaryDirectory() as d: + g = lambda *a: subprocess.run( + ["git", "-C", d, "-c", "user.name=t", "-c", "user.email=t@t", *a], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + g("init", "-q", "-b", "main") + g("commit", "-q", "--allow-empty", "-m", "fix: real (#9)") + o = g("rev-parse", "HEAD") + g("update-ref", "refs/remotes/origin/main", o) + cwd = os.getcwd() + os.chdir(d) + self.addCleanup(os.chdir, cwd) + pr = {"headRefName": f"backport/0.187/{o[:7]}"} + edited = f"fix: edited by hand\n\n(cherry picked from commit {o[:12]})" + with mock.patch.object(bm, "head_messages", lambda pr: [edited]): + self.assertEqual( + bm.first_pick(pr), + (f"fix: real (#9)\n\n(cherry picked from commit {o})", [o]), + ) + with mock.patch.object( + bm, + "head_messages", + lambda pr: ["fix: x\n\n(cherry picked from commit " + "d" * 40 + ")"], + ): + self.assertEqual( + bm.first_pick({"headRefName": "backport/0.187/ddddddd"}), ("", []) + ) # not on main + + +class Summary(unittest.TestCase): + def test_blocks(self): + rec = lambda result, sha="6" * 40, reason=None: { + "result": result, + "base": "release/0.189", + "sha": sha, + "reason": reason, + } + with mock.patch.dict(os.environ, clear=True): + self.assertEqual( + bm.summary(246, ME, rec("declined 6666666", reason="not on 0.189")), + [ + "**Backport PR** #246 on `release/0.189`: **declined** as `6666666`", + f'Reason: "not on 0.189" (@{ME})', + "Undo: reopen #246 and merge it.", + ], + ) + self.assertEqual( + bm.summary(246, ME, rec("merged")), + [ + f"**Backport PR** #246 on `release/0.189`: **merged** as `6666666` (@{ME})", + "Undo: revert `6666666` on `release/0.189`.", + ], + ) + self.assertEqual( + bm.summary(246, ME, rec("skipped: already on release/0.189 as 6666666")), + [ + "**Backport PR** #246 on `release/0.189`: **skipped**: already on `release/0.189` as `6666666`" + ], + ) + self.assertEqual(len(bm.summary(246, ME, rec("reopened", sha=None))), 2) + + +class SquashRetry(unittest.TestCase): + PR = {"number": 4, "headRefOid": "a" * 40} + MOVED = RuntimeError( + "gh api -X (exit 1): gh: Base branch was modified. Review and try the merge again. (HTTP 405)" + ) + + def run_squash(self, answers): + calls, waits = [], [] + + def gh(*args, **kw): + calls.append(args) + a = answers.pop(0) + if isinstance(a, Exception): + raise a + + with mock.patch.object(bm, "gh", gh): + return ( + bm.squash("o/r", self.PR, "t\n\nb", sleep=waits.append), + len(calls), + waits, + ) + + def test_base_modified_is_retried(self): + self.assertEqual(self.run_squash([self.MOVED, None]), (True, 2, [5])) + + def test_base_modified_three_times_raises(self): + with self.assertRaisesRegex(RuntimeError, "Base branch was modified"): + self.run_squash([self.MOVED] * 3) + + def test_pending_check_is_not_retried(self): + self.assertEqual( + self.run_squash([RuntimeError("Required status check is expected")]), + (False, 1, []), + ) + + def test_other_error_is_not_retried(self): + with self.assertRaisesRegex(RuntimeError, "HTTP 409"): + self.run_squash([RuntimeError("Head branch was modified (HTTP 409)"), None]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/test_convgate.py b/.github/release/test_convgate.py new file mode 100644 index 0000000000..f1c0559653 --- /dev/null +++ b/.github/release/test_convgate.py @@ -0,0 +1,946 @@ +"""Synthetic-repo tests for convgate (2b-2d revert of a pick = decline; 2g-2h DECLINED rows and decline_records; 5 a plain fix is owed to a stabilized line; 7a-7e MISSING messages; 7f-7g a promoted stable tag is read before its GA; 9a-9h checks table, release-branch-only window, count line, next steps; 10a-10d OSS pins). Run: python test_convgate.py""" + +import contextlib +import datetime as dt +import io +import json +import os +import tempfile +import unittest +from unittest import mock + +os.environ["GITHUB_REPOSITORY"] = ( + "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets +) +import convgate as cg +import relmgr as rm +import relstate as rs + +os.environ.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") +os.environ.pop("GITHUB_OUTPUT", None) +os.environ.pop("GITHUB_STEP_SUMMARY", None) +os.environ.pop("OSS_REPO", None) +NOW = dt.datetime(2026, 9, 29, tzinfo=dt.UTC) + + +class Gate(unittest.TestCase): + """Row 1 base: 1.5 and 1.6 cut, then C lands on main and is picked to 1.5 only; cloud RCs on both.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.repo, self.reg = self._tmp.name, {} + rm.git(self.repo, "init", "-q", "-b", "main") + self.change("base", None) # unborn main: nothing to check out + for line in ("1.5", "1.6"): + self.g("branch", f"release/{line}", "main") + self.change(f"work on {line}", f"release/{line}") + self.c = self.change("C: the fix", "main") + self.pick(self.c, "release/1.5") + self.push("cloud/release", "v1.5.0-rc.1", "release/1.5") + self.push("cloud/release", "v1.6.0-rc.1", "release/1.6") + + def g(self, *args): + return rm.git( + self.repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args + ) + + def change(self, msg, branch): + if branch: + self.g("checkout", "-q", branch) + with open( + os.path.join(self.repo, f"{msg.split(':')[0].replace(' ', '_')}.txt"), "w" + ) as f: + f.write(msg) + self.g("add", "-A") + self.g("commit", "-q", "-m", msg) + return self.g("rev-parse", "HEAD") + + def pick(self, sha, branch): + self.g("checkout", "-q", branch) + self.g("cherry-pick", "-x", sha) + + def decline(self, sha, branch): + self.g("checkout", "-q", branch) + self.g( + "commit", + "-q", + "--allow-empty", + "-m", + f"decline C\n\n(cherry picked from commit {sha})", + ) + + def revert(self, sha, branch, short=False): + """The Revert button's commit on `branch`; short=True names the reverted commit abbreviated.""" + self.g("checkout", "-q", branch) + self.g("revert", "--no-commit", sha) + self.g( + "commit", + "-q", + "-m", + f'Revert "x"\n\nThis reverts commit {sha[:7] if short else sha}.', + ) + return self.g("rev-parse", "HEAD") + + def tip(self, branch): + return self.g("rev-parse", branch) + + def push(self, artifact, tag, ref): + self.g("tag", tag, ref) + self.reg.setdefault(artifact, {})[tag] = { + "digest": f"sha256:{len(self.reg):064x}{tag}", + "pushed_at": "2026-09-28T00:00:00Z", + } + + def run_gate( + self, + branch, + target="cloud", + annotate="warning", + prs=None, + promote_ref=None, + failures=None, + ): + """prs: {conflict PR head: (number, url)}, failures: {(source PR, line): run url}; injected (no network).""" + state = rs.derive(self.repo, NOW, lambda a: self.reg.get(a, {})) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + code, out, self.lines = cg.gate( + self.repo, + branch, + target, + state, + promote_ref=promote_ref, + annotate=annotate, + lookup=(prs or {}).get, + failure=lambda n, b: (failures or {}).get((n, b)), + ) + self.printed = buf.getvalue() + return code, "\n".join(out) + + def test_1_both_directions(self): + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 2) + self.assertIn(", 1 missing,", out) + code, out = self.run_gate("release/1.6") + self.assertEqual(code, 2) + self.assertIn("**release/1.5** (older", out) + + def test_2_decline_converges(self): + self.decline(self.c, "release/1.6") + for b in ("release/1.5", "release/1.6"): + code, out = self.run_gate(b) + self.assertEqual(code, 0, out) + self.assertIn( + "1 picks, 0 in the RC, 1 declined on release/1.6, 1 unclassifiable", out + ) + + def test_2b_revert_of_pick_declines_on_newer(self): + self.pick(self.c, "release/1.6") + r = self.revert(self.tip("release/1.6"), "release/1.6") + self.assertEqual(rm.classify_commit(self.repo, r), "declined") + self.assertEqual(rm.declined_originals(self.repo, "release/1.6"), {self.c}) + for b in ("release/1.5", "release/1.6"): + code, out = self.run_gate(b) + self.assertEqual(code, 0, out) + self.assertIn( + "1 picks, 0 in the RC, 1 declined on release/1.6, 1 unclassifiable", out + ) + + def test_2c_revert_of_pick_on_older_owes_nothing(self): + r = self.revert(self.tip("release/1.5"), "release/1.5", short=True) + self.assertEqual(rm.classify_commit(self.repo, r), "declined") + rows = rm.convergence_report(self.repo, "release/1.5", "release/1.6") + self.assertEqual( + [st for _, st in rows], ["UNCLASSIFIABLE"] + ) # the fixture's plain work commit only + for b in ("release/1.5", "release/1.6"): + code, out = self.run_gate(b) + self.assertEqual(code, 0, out) + + def test_2e_pick_after_decline_ships(self): + """A decline undone by merging the reopened PR: the newest trailer wins, the fix counts as picked.""" + self.decline(self.c, "release/1.6") + self.pick(self.c, "release/1.6") + self.assertEqual( + rm.trailer_states(self.repo, "release/1.6"), {self.c: "cherry-pick"} + ) + self.assertEqual(rm.declined_originals(self.repo, "release/1.6"), set()) + for b in ("release/1.5", "release/1.6"): + code, out = self.run_gate(b) + self.assertEqual(code, 0, out) + self.assertIn("1 picks, 1 in the RC, 1 unclassifiable", out) + + def test_2f_pick_after_decline_past_ga_still_owed(self): + """Newer is GA at a tag without the fix; its decline was undone by a later pick: older's promote waits for + newer's next patch instead of reading the stale decline as converged. + """ + self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.g("tag", "v1.6.0", "v1.6.0-rc.1") + self.decline(self.c, "release/1.6") + self.pick(self.c, "release/1.6") + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 2, out) + self.assertIn("into `refs/tags/v1.6.0`", out) + + def test_2g_declined_row(self): + """A decline converges (exit 0) and its row names the reason and the decline commit.""" + self.g("checkout", "-q", "release/1.6") + self.g( + "commit", + "-q", + "--allow-empty", + "-m", + f"x\n\n(cherry picked from commit {self.c})\n\nDeclined: not needed on 1.6", + ) + d = self.tip("release/1.6") + pick = self.g("rev-list", "-1", "release/1.5") + for b in ("release/1.5", "release/1.6"): + code, out = self.run_gate(b) + self.assertEqual(code, 0, out) + self.assertIn( + "1 picks, 0 in the RC, 1 declined on release/1.6, 1 unclassifiable", out + ) + self.assertIn( + f"- `{pick[:12]}` DECLINED on release/1.6 original {self.c[:12]}: not needed on 1.6 ({d[:7]})", + out, + ) + + def test_2g2_declined_on_older(self): + """A decline on the older line owes the newer line nothing: no row, nothing counted.""" + self.g("checkout", "-q", "release/1.5") + self.g( + "commit", + "-q", + "--allow-empty", + "-m", + f"x\n\n(cherry picked from commit {self.c})\n\nDeclined: not needed on 1.5", + ) + d = self.tip("release/1.5") + for b in ("release/1.5", "release/1.6"): + code, out = self.run_gate(b) + self.assertEqual(code, 0, out) + self.assertNotIn("DECLINED", out) + self.assertNotIn(d[:12], out) + self.assertIn("0 in the RC, 1 unclassifiable\n", out) + + def test_2h_decline_records(self): + """Empty decline: its last `Declined:` line (subject without one); revert of a pick: the revert's subject; + a later pick clears the record. + """ + self.decline(self.c, "release/1.6") + d = self.tip("release/1.6") + self.assertEqual( + rm.decline_records(self.repo, "release/1.6"), {self.c: (d, "decline C")} + ) + self.g( + "commit", + "-q", + "--allow-empty", + "-m", + f"x\n\n(cherry picked from commit {self.c})\n\nDeclined: first\nDeclined: second", + ) + self.assertEqual( + rm.decline_records(self.repo, "release/1.6"), + {self.c: (self.tip("release/1.6"), "second")}, + ) + self.pick(self.c, "release/1.6") + self.assertEqual(rm.decline_records(self.repo, "release/1.6"), {}) + r = self.revert(self.tip("release/1.6"), "release/1.6") + self.assertEqual( + rm.decline_records(self.repo, "release/1.6"), {self.c: (r, 'Revert "x"')} + ) + self.assertEqual( + set(rm.decline_records(self.repo, "release/1.6")), + rm.declined_originals(self.repo, "release/1.6"), + ) + + def test_2d_revert_of_non_pick_is_invalid(self): + work = self.tip("release/1.6") # the fixture's "work on 1.6": not a pick + r = self.revert(work, "release/1.6") + self.assertEqual(rm.classify_commit(self.repo, r), "invalid") + self.assertIn(r, rm.check_branch(self.repo, "release/1.6")) + + def test_8a_backport_pr_judged_as_its_squash(self): + """A plain resolution commit after the pick passes on a backport/* PR; any other PR still classifies all.""" + self.g("checkout", "-q", "-b", "backport/1.6/abcdef0", "release/1.6") + self.g("cherry-pick", "-x", self.c) + first = self.tip("backport/1.6/abcdef0") + self.change("fixup: resolve", "backport/1.6/abcdef0") + self.assertEqual( + rm.pr_commits(self.repo, "release/1.6", "backport/1.6/abcdef0"), [first] + ) + both = rm.pr_commits(self.repo, "release/1.6", "dev/x") + self.assertEqual(len(both), 2) + self.assertEqual(rm.classify_commit(self.repo, both[1]), "invalid") + + def test_8b_conflict_markers(self): + self.assertEqual(rm.conflict_markers(self.repo), []) + self.g("checkout", "-q", "release/1.6") + with open(os.path.join(self.repo, "f.txt"), "w") as f: + f.write("a\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> c (fix)\n") + with open(os.path.join(self.repo, "g.txt"), "w") as f: + f.write("<<<<<<<< not a marker\n=======\n") + self.g("add", "f.txt", "g.txt") + self.g("commit", "-q", "-m", "markers") + self.assertEqual(rm.conflict_markers(self.repo), ["f.txt"]) + + def test_7a_older_promote_names_conflict_pr(self): + c7 = self.c[:7] + code, out = self.run_gate( + "release/1.5", annotate="error", prs={f"backport/1.6/{c7}": (12, "u")} + ) + self.assertEqual(code, 2) + msg = ( + f"Cannot promote release/1.5 for cloud: fix {c7} is not on newer release/1.6 yet. " + "Conflict PR #12 on release/1.6: resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." + ) + self.assertIn(f"::error::{msg}", self.printed) + self.assertIn(msg, out) + + def test_7b_newer_promote_conflict_pr_on_itself(self): + c7 = self.c[:7] + code, _ = self.run_gate( + "release/1.6", annotate="error", prs={f"backport/1.6/{c7}": (12, "u")} + ) + self.assertEqual(code, 2) + self.assertIn( + f"::error::Cannot promote release/1.6 for cloud: fix {c7} is on older release/1.5 but not on " + "this line. Conflict PR #12 is on this branch: resolve, get one approval and merge, or close the PR to decline (the close comment is the reason).", + self.printed, + ) + + def test_7c_no_pr_fallback_warns(self): + d = self.change("D: another fix (#9)", "main") + self.decline(self.c, "release/1.6") # leave D as the only MISSING row + self.pick(d, "release/1.5") + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 2) + msg = ( + f"Promote of release/1.5 for cloud will be blocked: fix {d[:7]} (PR #9) is not on newer release/1.6 " + f"yet. No Backport PR found for it: run Backport for {d[:7]}, or decline it on release/1.6." + ) + self.assertIn(f"::warning::{msg}", self.printed) + self.assertNotIn("::error::", self.printed) + # Backport's push-failure comment on PR #9 names the cause and the rerun instead; another line's does not + _, out = self.run_gate( + "release/1.5", failures={(9, "release/1.6"): "https://x/runs/5"} + ) + self.assertIn( + f"(PR #9) is not on newer release/1.6 yet. Backport for {d[:7]} to release/1.6 failed: push " + f"rejected (run https://x/runs/5). Rerun Backport for {d[:7]} on release/1.6.", + out, + ) + _, out = self.run_gate( + "release/1.5", failures={(9, "release/1.7"): "https://x/runs/5"} + ) + self.assertIn(msg, out) + + def test_7d_on_newer_tip_not_shipped(self): + self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"][ + "v1.6.0-rc.1" + ] # 1.6 GA at v1.6.0 + self.g("tag", "v1.6.0", "v1.6.0-rc.1") + self.pick(self.c, "release/1.6") # on the tip after GA, so not in v1.6.0 + code, _ = self.run_gate("release/1.5") + self.assertEqual(code, 2) + self.assertIn( + f"fix {self.c[:7]} is on release/1.6 but not in its latest stable v1.6.0: " + "promote a release/1.6 patch first.", + self.printed, + ) + + def test_7e_on_promoted_tip_not_in_its_rc(self): + self.pick(self.c, "release/1.6") # on 1.6's tip after v1.6.0-rc.1 was cut + code, _ = self.run_gate( + "release/1.6", annotate="error", promote_ref="v1.6.0-rc.1" + ) + self.assertEqual(code, 2) + self.assertIn( + f"::error::Cannot promote release/1.6 for cloud: fix {self.c[:7]} is on release/1.6 " + "but not in RC v1.6.0-rc.1: take a new RC build first.", + self.printed, + ) + + def onprem(self): + """On-prem RCs on both lines (the cloud RC commits), then on-prem Promote of 1.6 at v1.6.0, chart not published.""" + for tag in ("v1.5.0-rc.1", "v1.6.0-rc.1"): + self.reg.setdefault("on-prem/release", {})[tag] = self.reg["cloud/release"][ + tag + ] + self.g("tag", "v1.6.0", "v1.6.0-rc.1") + self.reg["on-prem/release"]["v1.6.0"] = self.reg["on-prem/release"]["v1.6.0-rc.1"] + + def test_7f_promoted_not_published_read_at_its_tag(self): + self.onprem() + self.pick( + self.c, "release/1.6" + ) # after the Promote: the tip has it, v1.6.0 does not + code, out = self.run_gate("release/1.5", "on-prem") + self.assertEqual(code, 2, out) + self.assertIn( + "**release/1.6** (newer, stabilizing for on-prem; `refs/heads/release/1.5` into `refs/tags/v1.6.0`)", + out, + ) + self.assertIn( + f"fix {self.c[:7]} is on release/1.6 but not in its latest stable v1.6.0: " + "promote a release/1.6 patch first.", + self.printed, + ) + + def test_7g_patch_promoted_read_over_published(self): + self.onprem() + self.reg["on-prem/chart"] = { + "v1.6.0": self.reg["on-prem/release"]["v1.6.0"] + } # Publish v1.6.0 + self.pick(self.c, "release/1.6") + self.g("tag", "v1.6.1-rc.1", "release/1.6") + self.g("tag", "v1.6.1", "release/1.6") + self.reg["on-prem/release"]["v1.6.1-rc.1"] = self.reg["on-prem/release"][ + "v1.6.1" + ] = { + "digest": "sha256:61", + "pushed_at": "2026-09-28T00:00:00Z", + } # patch promoted, not published + code, out = self.run_gate("release/1.5", "on-prem") + self.assertEqual(code, 0, out) + self.assertIn( + "(newer, stabilized for on-prem; `refs/heads/release/1.5` into `refs/tags/v1.6.1`)", + out, + ) + + def checks(self, out, order=(), split=True): + table, detail = cg.checks(self.lines, list(order), out.splitlines(), split=split) + self.assertEqual(detail[0], "
Gate detail") + self.assertIn(out.splitlines()[0], detail) + return table + + def test_9a_checks_clean(self): + self.decline(self.c, "release/1.6") + _, out = self.run_gate("release/1.5") + self.assertEqual( + self.checks(out)[2:], + [ + "| ✅ fixes match newer lines | `release/1.6` 1 picks, 0 in the RC, 1 declined on release/1.6, 1 unclassifiable | - |", + "| ✅ fixes match older lines | no other supported line | - |", + "| ✅ promote order | no older line is waiting to go GA | - |", + "| ✅ release-branch-only to check | none | - |", + ], + ) + self.assertEqual( + self.checks(out, split=False)[2], + "| ✅ fixes match the other lines | `release/1.6` 1 picks, 0 in the RC, 1 declined on release/1.6, " + "1 unclassifiable | - |", + ) + + def test_9b_checks_missing_with_conflict_pr(self): + _, out = self.run_gate( + "release/1.6", prs={f"backport/1.6/{self.c[:7]}": (12, "u")} + ) + table = self.checks( + out, + [ + "release/1.5 is not GA for cloud yet: promote and publish it first, retire it, " + "or force with a reason" + ], + ) + self.assertEqual( + table[3], + "| ⛔ fixes match older lines | `release/1.5` 1 picks, 0 in the RC, 1 missing, 1 unclassifiable | merge or close " + f"conflict PR {rm.pr_link(12)}, then Take RC again |", + ) + self.assertEqual( + table[4], + "| ⛔ promote order | release/1.5 is not GA for cloud yet | " + "promote or retire `release/1.5` |", + ) + _, out = self.run_gate( + "release/1.5", prs={f"backport/1.6/{self.c[:7]}": (12, "u")} + ) # the PR is on newer + self.assertIn( + f"merge or close conflict PR {rm.pr_link(12)} on `release/1.6` |", + self.checks(out)[2], + ) + + def rbo( + self, title, date="2030-01-01T00:00:00Z", path="pin.txt", branch="release/1.5" + ): + """A Release-branch-only squash on `branch` committed at `date`, changing `path`.""" + self.g("checkout", "-q", branch) + os.makedirs(os.path.join(self.repo, os.path.dirname(path)), exist_ok=True) + with open(os.path.join(self.repo, path), "a") as f: + f.write(title) + self.g("add", path) + with mock.patch.dict( + os.environ, GIT_COMMITTER_DATE=date + ): # a squash lists the bot's commit and keeps + self.g( + "commit", + "-q", + "-m", + f"{title}\n\n* Release-branch-only: UN-37\n\nRelease-branch-only: UN-37", + ) + return self.tip(branch) + + def test_9c_checks_rbo_absent(self): + self.decline(self.c, "release/1.6") + rbo = self.rbo("pin the base image") # its trailer twice: one ticket + _, out = self.run_gate("release/1.5") + what = f"UN-37 pin the base image ({rm.commit_link(rbo)})" + self.assertIn(f"- `{rbo[:12]}` RBO-ABSENT {what}", out) + self.assertNotIn("RBO-ABSENT original", out) + self.assertEqual( + self.checks(out)[5], + f"| ⚠ release-branch-only to check | on `release/1.5`: {what}, not on " + "`release/1.6` | check whether the newer line needs its own equivalent |", + ) + # a title that already starts with the ticket names it once + rbo = self.rbo("UN-37 [MISC] pin the base image") + _, out = self.run_gate("release/1.5") + self.assertIn( + f"- `{rbo[:12]}` RBO-ABSENT UN-37 [MISC] pin the base image ({rm.commit_link(rbo)})\n", + out, + ) + + def test_9e_rbo_window(self): + self.decline(self.c, "release/1.6") + old = self.rbo( + "before the cut", "2020-01-01T00:00:00Z" + ) # 1.6 has no GA: its cut is the window + new = self.rbo("after the cut") + workflow = self.rbo( + "seed a workflow", path=".github/workflows/x.yaml" + ) # nothing outside .github/ + _, out = self.run_gate("release/1.5") + self.assertIn(" · 1 release-branch-only to check\n", out) + self.assertIn(new[:12], out) + self.assertNotIn(old[:12], out) + self.assertNotIn(workflow[:12], out) + # 1.6 GA for cloud: its GA tag's date is the window + self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + with mock.patch.dict(os.environ, GIT_COMMITTER_DATE="2031-01-01T00:00:00Z"): + self.g("tag", "-a", "v1.6.0", "-m", "promote", "v1.6.0-rc.1") + later = self.rbo("after the GA", "2032-01-01T00:00:00Z") + _, out = self.run_gate("release/1.5") + self.assertIn(" · 1 release-branch-only to check\n", out) + self.assertIn(later[:12], out) + self.assertNotIn(new[:12], out) + + def test_9f_checks_name_lines_as_needed(self): + self.reg["cloud/release"]["v1.5.0"] = self.reg["cloud/release"][ + "v1.5.0-rc.1" + ] # 1.5 GA: stabilized + self.g("tag", "v1.5.0", "v1.5.0-rc.1") + self.decline(self.c, "release/1.6") + _, out = self.run_gate("release/1.6") + self.assertEqual( + self.checks(out)[3], + "| ✅ fixes match older lines | 1 older line converged (1 picks) | - |", + ) + self.assertIn("**release/1.5** (older", out) # the detail keeps every line + self.rbo("pin the base image") + _, out = self.run_gate("release/1.6") + self.assertIn( + "| ✅ fixes match older lines | `release/1.5` 1 picks,", self.checks(out)[3] + ) + + def test_9g_count_line(self): + self.pick(self.c, "release/1.6") + _, out = self.run_gate("release/1.5") + self.assertIn( + "**release/1.6** (newer, stabilizing for cloud; `refs/heads/release/1.5` into `refs/heads/release/1.6`): " + "1 picks, 1 in the RC, 1 unclassifiable\n", + out, + ) + _, out = self.run_gate( + "release/1.6", promote_ref="v1.6.0-rc.1" + ) # the RC being promoted: still "the RC" + self.assertIn( + "into `refs/tags/v1.6.0-rc.1`): 1 picks, 0 in the RC, 1 missing, 1 unclassifiable\n", + out, + ) + self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.g( + "tag", "v1.6.0", "release/1.6" + ) # 1.6 GA at a tag with the pick: named by its tag + self.rbo("pin the base image") + _, out = self.run_gate("release/1.5") + self.assertIn( + "**release/1.6** (newer, stabilized for cloud; `refs/heads/release/1.5` into `refs/tags/v1.6.0`): " + "1 picks, 1 in v1.6.0, 1 unclassifiable · 1 release-branch-only to check\n", + out, + ) + self.assertIn( + "| `release/1.6` 1 picks, 1 in v1.6.0, 1 unclassifiable · 1 release-branch-only to check |", + self.checks(out)[2], + ) + self.push( + "cloud/release", "v1.6.1-rc.1", "release/1.6" + ) # a patch RC of a GA line: "the RC" again + _, out = self.run_gate("release/1.6", promote_ref="v1.6.1-rc.1") + self.assertIn( + "into `refs/tags/v1.6.1-rc.1`): 1 picks, 1 in the RC, 1 unclassifiable · 1 release-branch-only " + "to check\n", + out, + ) + + def test_9h_next_steps(self): + """The do-this items in table order, each once; the table cells are built from the same list.""" + _, out = self.run_gate( + "release/1.6", prs={f"backport/1.6/{self.c[:7]}": (12, "u")} + ) + self.lines.append(dict(self.lines[0])) # a second line with the same todo + self.lines.append( + dict(self.lines[0], todo=["take a new RC build first."]) + ) # a message's full stop goes + order = [ + "release/1.5 is not GA for cloud yet: promote and publish it first, retire it, or force with a reason" + ] + todo = f"merge or close conflict PR {rm.pr_link(12)}, then Take RC again" + self.assertEqual( + cg.next_steps(self.lines, order), + [todo, "take a new RC build first", "promote or retire `release/1.5`"], + ) + self.assertEqual( + cg.next_steps(self.lines, []), [todo, "take a new RC build first"] + ) + table = self.checks(out, order) + self.assertTrue( + table[3].endswith(f"| {todo}
take a new RC build first |"), table[3] + ) + self.assertTrue( + table[4].endswith("| promote or retire `release/1.5` |"), table[4] + ) + + def test_9d_main_blocks_on_promote_order(self): + self.decline( + self.c, "release/1.6" + ) # no MISSING row: only 1.5, not GA yet, holds 1.6 back + path, md = ( + os.path.join(self.repo, "state.json"), + os.path.join(self.repo, "out.md"), + ) + with open(path, "w") as f: + json.dump(rs.derive(self.repo, NOW, lambda a: self.reg.get(a, {})), f) + with ( + contextlib.redirect_stdout(io.StringIO()), + mock.patch.object(rs, "PINS", None), + ): + code = cg.main( + [ + "--repo", + self.repo, + "--branch", + "release/1.6", + "--target", + "cloud", + "--state", + path, + "--out", + md, + ] + ) + self.assertEqual(code, 2) + with open(md) as f: + self.assertIn( + "| ⛔ promote order | release/1.5 is not GA for cloud yet |", f.read() + ) + + def test_3_scope_is_the_target(self): + self.push("on-prem/release", "v1.5.0-rc.2", "release/1.5") + code, out = self.run_gate("release/1.5", "on-prem") + self.assertEqual(code, 0) + self.assertIn("scanned branches: none", out) + self.assertEqual(self.run_gate("release/1.5", "cloud")[0], 2) + + def test_4_ga_branch_scanned_at_its_ga_tag(self): + self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"][ + "v1.6.0-rc.1" + ] # promote, same digest + self.g("tag", "v1.6.0", "v1.6.0-rc.1") + self.pick(self.c, "release/1.6") # after GA: the tip has it, v1.6.0 does not + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 2, out) + self.assertIn("into `refs/tags/v1.6.0`", out) + self.decline(self.c, "release/1.6") # newer_branch carries the decline + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 0, out) + + def test_5_plain_fix_owed_to_stabilized(self): + self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.g("tag", "v1.6.0", "v1.6.0-rc.1") # 1.6 GA = stabilized for cloud + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 2, out) + self.assertIn( + f"- `{self.tip('release/1.5')[:12]}` MISSING original {self.c[:12]}", out + ) + + def test_6_shipped_prs(self): + """shipped_prs: a pick counts, a decline and a main-only commit do not; C has no (#n), so lookup decides.""" + d, e = ( + self.change("D: picked (#7)", "main"), + self.change("E: declined (#8)", "main"), + ) + self.change("F: main only (#9)", "main") + self.pick(d, "release/1.5") + self.g( + "commit", + "-q", + "--allow-empty", + "-m", + f"decline E\n\n(cherry picked from commit {e})", + ) + self.g("tag", "v1.5.1", "release/1.5") + self.g("checkout", "-q", "main") + self.assertEqual( + rm.shipped_prs(self.repo, ["v1.5.1"], lookup={self.c: 5}.get), {5, 7} + ) + with self.assertRaisesRegex(RuntimeError, "no PR number"): + rm.shipped_prs(self.repo, ["v1.5.1"]) + self.assertEqual(rm.shipped_prs(self.repo, []), set()) + + def test_7_shipped_prs_newest_trailer(self): + """shipped_prs: newest trailer wins. A pick backed out by a later decline or revert is not shipped; a pick + after a decline is. + """ + d, r, p = ( + self.change(f"{x}: fix (#{n})", "main") + for x, n in (("D", 21), ("R", 22), ("P", 23)) + ) + self.pick(d, "release/1.5") + self.decline(d, "release/1.5") # (a) pick, then decline + self.pick(r, "release/1.5") + self.revert( + self.tip("release/1.5"), "release/1.5" + ) # (b) pick, then the Revert button's commit + self.decline(p, "release/1.5") + self.pick(p, "release/1.5") # (c) decline, then pick + self.g("checkout", "-q", "main") + self.assertEqual( + rm.shipped_prs(self.repo, ["release/1.5"], lookup={self.c: 5}.get), {5, 23} + ) + self.assertEqual( + rm.shipped_prs( + self.repo, ["release/1.5", "release/1.6"], lookup={self.c: 5}.get + ), + {5, 23}, + ) + + def test_7b_shipped_prs_line_picks(self): + """shipped_prs(line=): the new line's picks of shipped originals drop too, by lookup (clean pick, original's + subject) or by their own (#n) (conflict squash); a pick of an original declined on the previous line, a + release-branch-only commit and line=None do not change. + """ + d, e = self.change("D: fix (#21)", "main"), self.change("E: fix (#22)", "main") + f = self.change("F: fix (#23)", "main") + self.pick(d, "release/1.5") + self.pick(f, "release/1.5") + self.decline(e, "release/1.5") + self.g("tag", "v1.5.1", "release/1.5") + self.pick(d, "release/1.6") + pd = self.tip( + "release/1.6" + ) # clean pick: subject keeps (#21); its PR is found by lookup + self.change( + "F_squash: resolved", None + ) # conflict pick, squashed under its PR's title; trailer abbreviated + self.g( + "commit", + "-q", + "--amend", + "-m", + f"UN-9 [FIX] F (#41)\n\n(cherry picked from commit {f[:12]})", + ) + self.pick(e, "release/1.6") + pe = self.tip("release/1.6") + self.change("RBO: line only (#50)", "release/1.6") + self.g("tag", "v1.6.0-rc.2", "release/1.6") + self.g("checkout", "-q", "main") + prs = {self.c: 5, pd: 40, pe: 42}.get + self.assertEqual(rm.shipped_prs(self.repo, ["v1.5.1"], lookup=prs), {5, 21, 23}) + self.assertEqual( + rm.shipped_prs(self.repo, ["v1.5.1"], lookup=prs, line="v1.6.0-rc.2"), + {5, 21, 23, 40, 41}, + ) + self.assertEqual( + rm.shipped_prs( + self.repo, ["v1.5.1"], lookup={self.c: 5}.get, line="v1.6.0-rc.2" + ), + {5, 21, 23, 41}, + ) # no lookup hit for the clean pick: its (#21) only, nothing raised + + +class OssPins(unittest.TestCase): + """Cloud 1.5 pins OSS ee-v1.5.1, which carries OSS fix F; cloud 1.6 pins ee-v1.6.0, cut before F.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.reg = {} + self.cloud, self.oss = (os.path.join(self._tmp.name, d) for d in ("cloud", "oss")) + for r in (self.cloud, self.oss): + os.mkdir(r) + rm.git(r, "init", "-q", "-b", "main") + self.commit(r, "base") + o = self.oss + self.g(o, "branch", "release/1.5", "main") + self.g(o, "branch", "release/1.6", "main") + self.f = self.commit(o, "F: oss fix (#7)", file="f.txt") + self.g(o, "checkout", "-q", "release/1.5") + self.g(o, "cherry-pick", "-x", self.f) + self.g(o, "tag", "ee-v1.5.1") + self.g(o, "checkout", "-q", "release/1.6") + self.commit(o, "release: ee-v1.6.0", bot=True) + self.g(o, "tag", "ee-v1.6.0") + self.g(o, "checkout", "-q", "main") + self.sync() + for line, pin in (("1.5", "ee-v1.5.1"), ("1.6", "ee-v1.6.0")): + self.g(self.cloud, "branch", f"release/{line}", "main") + self.repin(line, pin, f"v{line}.0-rc.1") + + def g(self, repo, *args): + return rm.git( + repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args + ) + + def commit(self, repo, msg, file="x.txt", text=None, bot=False): + with open(os.path.join(repo, file), "w") as f: + f.write(text or msg) + self.g(repo, "add", file) + who = ["-c", "user.name=bot", "-c", f"user.email={rm.BOT_EMAIL}"] if bot else [] + rm.git( + repo, + *(who or ["-c", "user.name=Dev", "-c", "user.email=dev@example.com"]), + "commit", + "-q", + "-m", + msg, + ) + return self.g(repo, "rev-parse", "HEAD") + + def sync(self): # the OSS clone's origin/* refs, as `gh repo clone` leaves them + for b in self.g( + self.oss, "for-each-ref", "--format=%(refname:short)", "refs/heads/" + ).split(): + self.g(self.oss, "update-ref", f"refs/remotes/origin/{b}", b) + + def repin(self, line, pin, rc): + self.g(self.cloud, "checkout", "-q", f"release/{line}") + self.commit( + self.cloud, + f"release: pin OSS {pin}", + "pyproject.toml", + f'unstract-platform-version = "{pin}"\n', + bot=True, + ) + self.g(self.cloud, "tag", rc) + self.g(self.cloud, "checkout", "-q", "main") + reg = self.reg.setdefault("cloud/release", {}) + reg[rc] = { + "digest": f"sha256:{len(reg):064x}", + "pushed_at": "2026-09-28T00:00:00Z", + } + + def run_gate(self, branch): + state = rs.derive(self.cloud, NOW, lambda a: self.reg.get(a, {})) + with contextlib.redirect_stdout(io.StringIO()): + code, out, lines = cg.gate( + self.cloud, + branch, + "cloud", + state, + annotate="error", + lookup=lambda h: None, + failure=lambda n, b: None, + oss_repo=self.oss, + ) + return code, "\n".join(out), lines + + def test_10a_missing_oss_fix_names_the_oss_line_then_the_rc(self): + code, out, lines = self.run_gate("release/1.6") + self.assertEqual(code, 2) + self.assertIn( + "Cannot promote release/1.6 for cloud: OSS fix " + + self.f[:7] + + " (OSS PR #7) is in release/1.5's " + "OSS pin ee-v1.5.1 but not in release/1.6's pin ee-v1.6.0: pick it onto OSS release/1.6 (OSS " + "Backport), then take a new RC of release/1.6.", + out, + ) + self.assertEqual( + [(x["line"], x["side"], x["summary"]) for x in lines if x.get("oss")], + [("release/1.5", "older", "1 OSS picks, 0 in ee-v1.6.0, 1 missing")], + ) + table, _ = cg.checks(lines, [], out.splitlines(), split=True) + self.assertTrue( + any(r.startswith("| ⛔ OSS pins match the other lines |") for r in table), + table, + ) + self.assertEqual( + table[3], + "| ✅ fixes match older lines | `release/1.5` 0 picks, 0 in the RC | - |", + ) # cloud rows: cloud only + # The fix reaches the OSS line: the pin, not OSS, is behind now. + self.g(self.oss, "checkout", "-q", "release/1.6") + self.g(self.oss, "cherry-pick", "-x", self.f) + self.g(self.oss, "checkout", "-q", "main") + self.sync() + code, out, _ = self.run_gate("release/1.5") # same scan from the older side + self.assertEqual(code, 2) + self.assertIn(": take a new RC of release/1.6 to pin OSS release/1.6's tip.", out) + + def test_10b_repinned_newer_line_converges(self): + self.g(self.oss, "checkout", "-q", "release/1.6") + self.g(self.oss, "cherry-pick", "-x", self.f) + self.g(self.oss, "tag", "ee-v1.6.1") + self.g(self.oss, "checkout", "-q", "main") + self.sync() + self.repin("1.6", "ee-v1.6.1", "v1.6.0-rc.2") + code, out, lines = self.run_gate("release/1.6") + self.assertEqual(code, 0, out) + self.assertEqual( + [x["summary"] for x in lines if x.get("oss")], ["1 OSS picks, 1 in ee-v1.6.1"] + ) + self.repin("1.6", "ee-v1.5.1", "v1.6.0-rc.3") # one shared pin: nothing to scan + self.assertEqual( + [x["summary"] for x in self.run_gate("release/1.6")[2] if x.get("oss")], + ["0 OSS picks, 0 in ee-v1.5.1"], + ) + + def test_10d_decline_on_the_oss_line_converges(self): + # Declined on OSS release/1.6 after ee-v1.6.0: the line owes nothing, though the pinned tag predates the decline. + self.g(self.oss, "checkout", "-q", "release/1.6") + self.g( + self.oss, + "commit", + "-q", + "--allow-empty", + "-m", + f"decline F\n\n(cherry picked from commit {self.f})", + ) + self.g(self.oss, "checkout", "-q", "main") + self.sync() + code, out, _ = self.run_gate("release/1.6") + self.assertEqual(code, 0, out) + + def test_10c_pinning_repo_needs_the_oss_clone(self): + with ( + contextlib.redirect_stdout(io.StringIO()) as buf, + contextlib.redirect_stderr(io.StringIO()) as err, + ): + code = cg.main( + ["--repo", self.cloud, "--branch", "release/1.6", "--target", "cloud"] + ) + self.assertEqual(code, 1) + self.assertIn( + "::error::convgate: RuntimeError: --oss-repo (or OSS_REPO) is required", + buf.getvalue(), + ) + self.assertIn("Traceback", err.getvalue()) # where it failed, not only what + self.g(self.oss, "tag", "-d", "ee-v1.6.0") + with self.assertRaisesRegex(RuntimeError, "no OSS tag ee-v1.6.0"): + self.run_gate("release/1.6") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/test_links.py b/.github/release/test_links.py new file mode 100644 index 0000000000..42d2afdf8b --- /dev/null +++ b/.github/release/test_links.py @@ -0,0 +1,43 @@ +import os +import unittest +from unittest import mock + +import relmgr as rm + +ACTIONS = { + "GITHUB_SERVER_URL": "https://github.com", + "GITHUB_REPOSITORY": "Zipstack/example", +} +WEB = "https://github.com/Zipstack/example" +SHA = "6635951" + "0" * 33 + + +class Links(unittest.TestCase): + def test_actions(self): + with mock.patch.dict(os.environ, ACTIONS): + self.assertEqual(rm.pr_link(246), f"[#246]({WEB}/pull/246)") + self.assertEqual(rm.commit_link(SHA), f"[`6635951`]({WEB}/commit/{SHA})") + self.assertEqual( + rm.tag_link("v0.192.0-rc.1"), + f"[`v0.192.0-rc.1`]({WEB}/releases/tag/v0.192.0-rc.1)", + ) + self.assertEqual(rm.run_link(9), f"[run]({WEB}/actions/runs/9)") + self.assertEqual( + rm.image_link("on-prem", "v0.192.0-rc.1"), + "[`on-prem/release:v0.192.0-rc.1`](https://github.com/orgs/Zipstack/packages/container/" + "package/example%2Fon-prem%2Frelease)", + ) + + def test_plain_outside_actions(self): + with mock.patch.dict(os.environ, clear=True): + self.assertEqual(rm.pr_link(246), "#246") + self.assertEqual(rm.commit_link(SHA), "`6635951`") + self.assertEqual(rm.tag_link("v0.192.0-rc.1"), "`v0.192.0-rc.1`") + self.assertEqual(rm.run_link(9), "run 9") + self.assertEqual( + rm.image_link("cloud", "v0.192.0-rc.1"), "`cloud/release:v0.192.0-rc.1`" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py new file mode 100644 index 0000000000..1e97980856 --- /dev/null +++ b/.github/release/test_oss.py @@ -0,0 +1,973 @@ +"""OSS line work by git on a clone of a bare origin: cut (refuses an existing line or used minor, recovers its own +re-run), rc (tip tag else a new ee- RC; plain v* ignored), promote (open conflict PRs and the gate block unless forced, re-runnable), retire +(archive tag; RC tags go without a GA), and a tag already on another commit fails. Run: python test_oss.py +""" + +import contextlib +import io +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +os.environ["GITHUB_REPOSITORY"] = ( + "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets +) +import oss +import relmgr as rm + +os.environ.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") +os.environ.pop("GITHUB_STEP_SUMMARY", None) + + +class Choose(unittest.TestCase): + def test_stable_else_newest_rc(self): + self.assertEqual(oss.choose(["ee-v1.7.0-rc.2", "ee-v1.7.0"]), "ee-v1.7.0") + self.assertEqual( + oss.choose(["ee-v1.7.0-rc.9", "ee-v1.7.0-rc.10"]), "ee-v1.7.0-rc.10" + ) + self.assertIsNone(oss.choose([])) + + +class Lines(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.origin, self.dev, self.oss, self.cloud = ( + os.path.join(tmp.name, d) for d in ("origin", "dev", "oss", "cloud") + ) + rm.git(tmp.name, "init", "-q", "--bare", "-b", "main", self.origin) + rm.git(tmp.name, "clone", "-q", self.origin, self.dev) + self.g(self.dev, "commit", "-q", "--allow-empty", "-m", "base") + self.g( + self.dev, "tag", "v1.7.3" + ) # a community release on main: never an OSS line's tag + self.g(self.dev, "push", "-q", "origin", "main", "v1.7.3") + self.clone() + self.gates = [] + self.gate = 0 + p = mock.patch.object( + oss, + "gate", + lambda o, b, ref, annotate: self.gates.append((b, ref, annotate)) + or self.gate, + ) + p.start() + self.addCleanup(p.stop) + self.prs, self.listed = ( + [], + [], + ) # open backport-conflict PRs of the OSS repo, and each listing's (slug, base, token) + p = mock.patch.object( + oss.bp, + "conflict_prs", + lambda *a: self.listed.append(a) + or [{"number": n, "baseRefName": b} for n, b in self.prs if b == a[1]], + ) + p.start() + self.addCleanup(p.stop) + + def g(self, repo, *args): + return rm.git( + repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args + ) + + def clone(self): # a fresh workflow run's OSS clone + rm.git( + os.path.dirname(self.oss), + "clone", + "-q", + "--no-checkout", + self.origin, + self.oss, + ) if not os.path.isdir(self.oss) else self.g( + self.oss, "fetch", "-q", "--prune", "--prune-tags", "--tags", "origin" + ) + + def remote(self, *args): + return self.g(self.origin, *args) + + def quiet(self, f, *args, **kw): + with contextlib.redirect_stderr(io.StringIO()): + return f(*args, **kw) + + def work(self, branch, msg): # a developer's commit on the OSS line, pushed + self.g(self.dev, "fetch", "-q", "origin") + self.g(self.dev, "checkout", "-q", "-B", branch, f"origin/{branch}") + self.g(self.dev, "commit", "-q", "--allow-empty", "-m", msg) + self.g(self.dev, "push", "-q", "origin", branch) + self.g(self.dev, "checkout", "-q", "main") + self.clone() + + def test_cut(self): + self.assertEqual( + self.quiet(oss.cut, self.oss, "1.7", "https://x.slack.com/archives/C1/p1"), + "ee-v1.7.0-rc.1", + ) + tip = self.remote("rev-parse", "release/1.7") + self.assertEqual(self.remote("rev-parse", "ee-v1.7.0-rc.1^{commit}"), tip) + self.assertEqual( + self.remote("log", "-1", "--format=%ae %s", tip), + f"{rm.BOT_EMAIL} release: ee-v1.7.0-rc.1", + ) + self.assertEqual( + self.remote("rev-parse", f"{tip}^"), self.remote("rev-parse", "main") + ) + self.assertIn( + "Slack: https://x.slack.com/archives/C1/p1", + self.remote("tag", "-l", "ee-v1.7.0-rc.1", "--format=%(contents)"), + ) + # A re-run of the same cut (its cloud branch push failed) finds its own result. + self.clone() + self.assertEqual(self.quiet(oss.cut, self.oss, "1.7"), "ee-v1.7.0-rc.1") + self.assertEqual(self.remote("rev-parse", "release/1.7"), tip) + # Any other existing line is refused: work on it, or a line cut by hand. + self.work("release/1.7", "oss work") + with self.assertRaisesRegex(RuntimeError, "OSS release/1.7 already exists"): + oss.cut(self.oss, "1.7") + self.g(self.dev, "push", "-q", "origin", "main:refs/heads/release/1.8") + self.clone() + with self.assertRaisesRegex(RuntimeError, "OSS release/1.8 already exists"): + oss.cut(self.oss, "1.8") + + def test_cut_refuses_a_used_minor(self): + self.remote( + "tag", "ee-v1.9.0", "main" + ) # a retired line that shipped: its GA tag stays + self.clone() + with self.assertRaisesRegex( + RuntimeError, r"OSS minor 1.9 already has tags \(ee-v1.9.0\)" + ): + oss.cut(self.oss, "1.9") + self.assertEqual( + self.quiet(oss.cut, self.oss, "1.7"), "ee-v1.7.0-rc.1" + ) # plain v1.7.3 does not count + + def test_rc(self): + with self.assertRaisesRegex(RuntimeError, "OSS release/1.7 does not exist"): + oss.rc(self.oss, "release/1.7") + self.quiet(oss.cut, self.oss, "1.7") + self.clone() + self.assertEqual( + self.quiet(oss.rc, self.oss, "release/1.7"), "ee-v1.7.0-rc.1" + ) # the tip's tag + self.work("release/1.7", "oss fix") + self.remote( + "tag", "v1.7.4", "release/1.7" + ) # a plain v* tag on the tip is not the line's + self.clone() + self.assertEqual(self.quiet(oss.rc, self.oss, "release/1.7"), "ee-v1.7.0-rc.2") + tip = self.remote("rev-parse", "release/1.7") + self.assertEqual(self.remote("rev-parse", "ee-v1.7.0-rc.2^{commit}"), tip) + self.assertEqual( + self.remote("log", "-1", "--format=%s", tip), "release: ee-v1.7.0-rc.2" + ) + self.assertEqual( + self.gates, + [ + ("release/1.7", "ee-v1.7.0-rc.1", "warning"), + ("release/1.7", "ee-v1.7.0-rc.2", "warning"), + ], + ) + self.gate = 2 # the preview never blocks + self.assertEqual(self.quiet(oss.rc, self.oss, "release/1.7"), "ee-v1.7.0-rc.2") + + def pin(self, tag, rc): + if not os.path.isdir(self.cloud): + rm.git(os.path.dirname(self.cloud), "init", "-q", "-b", "main", self.cloud) + with open(os.path.join(self.cloud, "pyproject.toml"), "w") as f: + f.write(f'unstract-platform-version = "{tag}"\n') + self.g(self.cloud, "add", "pyproject.toml") + self.g(self.cloud, "commit", "-q", "-m", f"pin {tag}") + self.g(self.cloud, "tag", rc) + + def test_promote(self): + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + self.gate = 2 + with self.assertRaisesRegex(RuntimeError, "blocks: MISSING picks"): + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + self.assertEqual(self.remote("tag", "-l", "ee-v1.7.0"), "") + self.assertEqual( + oss.promote( + self.cloud, self.oss, "v1.7.0-rc.1", force=True, reason="known gap" + ), + "ee-v1.7.0", + ) + self.assertEqual( + self.remote("rev-parse", "ee-v1.7.0^{commit}"), + self.remote("rev-parse", "ee-v1.7.0-rc.1^{commit}"), + ) + msg = self.remote("tag", "-l", "ee-v1.7.0", "--format=%(contents)") + self.assertIn("promote: ee-v1.7.0-rc.1 -> ee-v1.7.0 (cloud v1.7.0-rc.1)", msg) + self.assertIn("forced: known gap", msg) + n = len(self.gates) + self.clone() # the other cloud product: finds the tag, skips the gate + self.assertEqual(oss.promote(self.cloud, self.oss, "v1.7.0-rc.1"), "ee-v1.7.0") + self.assertEqual(len(self.gates), n) + self.pin("ee-v1.7.0", "v1.7.1-rc.1") # a stable pin needs no promote + self.assertEqual(oss.promote(self.cloud, self.oss, "v1.7.1-rc.1"), "ee-v1.7.0") + self.gate = 1 + self.work("release/1.7", "oss fix") + self.quiet(oss.rc, self.oss, "release/1.7") + self.pin("ee-v1.7.1-rc.1", "v1.7.1-rc.2") + with self.assertRaisesRegex(RuntimeError, "did not run"): + oss.promote(self.cloud, self.oss, "v1.7.1-rc.2", force=True, reason="x") + + def test_promote_refuses_a_downgrade(self): + self.quiet(oss.cut, self.oss, "1.7") + self.work("release/1.7", "oss fix") + self.quiet(oss.rc, self.oss, "release/1.7") + self.remote("tag", "ee-v1.7.0", "ee-v1.7.0-rc.2") # the later RC shipped + self.clone() + with self.assertRaisesRegex( + ValueError, + "ee-v1.7.0-rc.1 is older than ee-v1.7.0: promoting it would be a downgrade", + ): + rm.promote_version(self.oss, "ee-v1.7.0-rc.1") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + with self.assertRaisesRegex( + RuntimeError, "^OSS ee-v1.7.0-rc.1 is older than ee-v1.7.0" + ): + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + self.assertEqual( + self.gates, [("release/1.7", "ee-v1.7.0-rc.2", "warning")] + ) # refused before its gate + + @mock.patch.dict(os.environ, {"OSS_TOKEN": "oss-token"}) + def test_promote_blocks_on_open_conflict_prs(self): + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + self.prs = [(4, "release/1.7"), (5, "release/1.6")] + with self.assertRaisesRegex( + RuntimeError, r"OSS release/1.7 has open backport conflict PRs #4: " + ): + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + self.assertEqual(self.listed, [(oss.OSS, "release/1.7", "oss-token")]) + self.assertEqual(self.remote("tag", "-l", "ee-v1.7.0"), "") + self.assertEqual( + oss.promote( + self.cloud, self.oss, "v1.7.0-rc.1", force=True, reason="ship it" + ), + "ee-v1.7.0", + ) + msg = self.remote("tag", "-l", "ee-v1.7.0", "--format=%(contents)") + self.assertIn("forced: ship it", msg) + self.assertIn("open conflict PRs: #4", msg) + + def test_promote_fails_closed_when_the_listing_fails(self): + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + with mock.patch.object( + oss.bp, "conflict_prs", side_effect=RuntimeError("gh pr list (exit 1)") + ): + with self.assertRaisesRegex(RuntimeError, "gh pr list"): + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + self.assertEqual(self.remote("tag", "-l", "ee-v1.7.0"), "") + with contextlib.redirect_stderr( + io.StringIO() + ) as err: # forced: promoted, the gap recorded on the tag + self.assertEqual( + oss.promote( + self.cloud, self.oss, "v1.7.0-rc.1", force=True, reason="x" + ), + "ee-v1.7.0", + ) + self.assertIn( + "::warning::Could not list open backport conflict PRs on OSS release/1.7", + err.getvalue(), + ) + msg = self.remote("tag", "-l", "ee-v1.7.0", "--format=%(contents)") + self.assertIn("forced: x", msg) + self.assertIn("conflict PRs unchecked: listing failed", msg) + + def test_tag_on_another_commit_fails(self): + self.quiet(oss.cut, self.oss, "1.7") + tip, main = (self.remote("rev-parse", r) for r in ("release/1.7", "main")) + self.assertFalse( + oss._tag(self.oss, "ee-v1.7.0-rc.1", tip) + ) # same commit: skipped + with self.assertRaisesRegex( + RuntimeError, f"OSS tag ee-v1.7.0-rc.1 is on {tip[:12]}, not on {main[:12]}" + ): + oss._tag(self.oss, "ee-v1.7.0-rc.1", main) + self.remote( + "tag", f"retired/1.7-{tip[:7]}", "main" + ) # the archive name, taken on another commit + self.clone() + with self.assertRaisesRegex(RuntimeError, f"OSS tag retired/1.7-{tip[:7]} is on"): + oss.retire(self.oss, "release/1.7", "x", dry_run=False) + self.assertEqual(self.remote("rev-parse", "release/1.7"), tip) # nothing deleted + + def test_retire(self): + self.quiet(oss.cut, self.oss, "1.7") + self.work("release/1.7", "oss fix") + self.quiet(oss.rc, self.oss, "release/1.7") + tip = self.remote("rev-parse", "release/1.7") + dry = oss.retire(self.oss, "release/1.7", "cut by mistake") + self.assertEqual( + dry, + [ + f"- OSS `release/1.7` (tip `{tip[:12]}`; no GA, RC tags deleted)", + "- OSS git tag `ee-v1.7.0-rc.1`", + "- OSS git tag `ee-v1.7.0-rc.2`", + f"- OSS tag `retired/1.7-{tip[:7]}`: cut by mistake", + ], + ) + self.assertEqual( + self.remote("tag", "-l", "ee-*").split(), ["ee-v1.7.0-rc.1", "ee-v1.7.0-rc.2"] + ) # dry run + self.assertEqual( + oss.retire(self.oss, "release/1.7", "cut by mistake", dry_run=False), dry + ) + self.assertEqual(self.remote("tag", "-l", "ee-*"), "") + self.assertEqual(self.remote("tag", "-l", "v*"), "v1.7.3") + self.assertEqual(self.remote("branch", "-l", "release/*"), "") + self.assertEqual( + self.remote("rev-parse", f"retired/1.7-{tip[:7]}^{{commit}}"), tip + ) + self.assertIn( + "retired release/1.7: cut by mistake", + self.remote("tag", "-l", "retired/*", "--format=%(contents)"), + ) + self.clone() # a re-run finds the line gone; the minor is free again + self.assertEqual( + oss.retire(self.oss, "release/1.7", "cut by mistake", dry_run=False), + [f"- OSS `release/1.7`: no branch (archived as `retired/1.7-{tip[:7]}`)"], + ) + self.assertEqual(self.quiet(oss.cut, self.oss, "1.7"), "ee-v1.7.0-rc.1") + self.clone() # the reused minor retires again: the old line's archive tag is not on its history + self.assertEqual( + len(oss.retire(self.oss, "release/1.7", "again", dry_run=False)), 3 + ) + self.assertEqual(len(self.remote("tag", "-l", "retired/1.7-*").split()), 2) + + def test_retire_refuses_a_line_moved_after_its_archive(self): + self.quiet(oss.cut, self.oss, "1.7") + tip = self.remote("rev-parse", "release/1.7") + self.remote( + "tag", f"retired/1.7-{tip[:7]}", tip + ) # an earlier run archived, then failed before the delete + self.work("release/1.7", "oss fix") + self.clone() + with self.assertRaisesRegex( + RuntimeError, f"moved to .* after retired/1.7-{tip[:7]} archived it" + ): + oss.retire(self.oss, "release/1.7", "again") + self.assertEqual( + self.remote("tag", "-l", "retired/*").split(), [f"retired/1.7-{tip[:7]}"] + ) + + def test_retire_keeps_a_shipped_line(self): + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + tip = self.remote("rev-parse", "release/1.7") + self.assertEqual( + oss.retire(self.oss, "release/1.7", "past end of support", dry_run=False), + [ + f"- OSS `release/1.7` (tip `{tip[:12]}`; has GA, tags kept)", + f"- OSS tag `retired/1.7-{tip[:7]}`: past end of support", + ], + ) + self.assertEqual( + self.remote("tag", "-l", "ee-*").split(), ["ee-v1.7.0", "ee-v1.7.0-rc.1"] + ) + self.assertEqual(self.remote("branch", "-l", "release/*"), "") + self.clone() + with self.assertRaisesRegex(RuntimeError, "OSS minor 1.7 already has tags"): + oss.cut(self.oss, "1.7") + + +class Gate(unittest.TestCase): + def test_runs_on_the_oss_clone_as_the_oss_repo(self): + # OSS 1.6 shipped with OSS fix F picked; OSS 1.7 lacks it. The gate reads only ee- tags and asks the OSS repo + # (not the cloud one) for Backport conflict PRs. + with tempfile.TemporaryDirectory() as d: + g = lambda *a: rm.git( + d, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a + ) + g("init", "-q", "-b", "main") + g("commit", "-q", "--allow-empty", "-m", "base") + g("branch", "release/1.6") + g("branch", "release/1.7") + with open(os.path.join(d, "f.txt"), "w") as fh: + fh.write("fix") + g("add", "f.txt") + g("commit", "-q", "-m", "F: oss fix (#7)") + f = g("rev-parse", "HEAD") + g("tag", "v1.7.3") # a community release: never the 1.7 line's + g("checkout", "-q", "release/1.6") + g("cherry-pick", "-x", f) + g("tag", "ee-v1.6.0-rc.1") + g("tag", "ee-v1.6.0") + g("checkout", "-q", "release/1.7") + g("commit", "-q", "--allow-empty", "-m", "release: ee-v1.7.0-rc.1") + g("tag", "ee-v1.7.0-rc.1") + for b in ("main", "release/1.6", "release/1.7"): + g("update-ref", f"refs/remotes/origin/{b}", b) + bin_ = os.path.join(d, "bin") + os.mkdir(bin_) + with open(os.path.join(bin_, "gh"), "w") as fh: + fh.write(f'#!/bin/sh\necho "$@" >> {d}/gh.log\necho "[]"\n') + os.chmod(os.path.join(bin_, "gh"), 0o755) + with mock.patch.dict( + os.environ, + PATH=f"{bin_}:{os.environ['PATH']}", + GITHUB_REPOSITORY="Zipstack/x", + ): + self.assertEqual(oss.gate(d, "release/1.7", "ee-v1.7.0-rc.1", "error"), 2) + with open(os.path.join(d, "gh.log")) as fh: + self.assertIn(f"-R {oss.OSS} --head backport/1.7/{f[:7]}", fh.read()) + + def test_usage_error_is_an_error_not_a_block(self): + # argparse exits 2 on a usage error, the code a MISSING block uses: only a block may be forced past. + with tempfile.TemporaryDirectory() as d: + fake = os.path.join(d, "convgate.py") + for script, want in ( + ("import sys; sys.exit(2)", 2), + ("import argparse; argparse.ArgumentParser().parse_args(['--nope'])", 1), + ("import sys; sys.exit(3)", 1), + ("pass", 0), + ): + with open(fake, "w") as fh: + fh.write(script) + with ( + mock.patch.object(oss, "HERE", d), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual( + oss.gate(d, "release/1.7", "ee-v1.7.0-rc.1", "error"), + want, + script, + ) + + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def inline(workflow, step): + """The inline python of a workflow step.""" + with open(os.path.join(HERE, "..", "workflows", workflow)) as fh: + wf = fh.read() + body = wf[ + wf.index("python3 - <<'PY'", wf.index(f"- name: {step}\n")) : + ] # the heredoc line may redirect + return "\n".join( + line[10:] for line in body.split("\n")[1 : body.split("\n").index(" PY")] + ) + + +@unittest.skipUnless( + os.path.exists(os.path.join(HERE, "..", "workflows", "retire.yaml")), + "the cloud repo's workflows", +) +class Workflow(unittest.TestCase): + """A cloud and an OSS origin with clones; a stub gh serves the cloud registry (reg.jsonl), the open + backport-conflict PRs of each repo (_.json; a `prfail` file fails the listing) and the PRs of a main + commit (pulls.json). gh.log: each pr list and commit PRs call, as its token. + """ + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.d = d = tmp.name + self.repos = {} + for name in ("cloud", "oss"): + origin, work = os.path.join(d, f"{name}.git"), os.path.join(d, name) + rm.git(d, "init", "-q", "--bare", "-b", "main", origin) + rm.git(d, "clone", "-q", origin, work) + self.g(work, "commit", "-q", "--allow-empty", "-m", "base") + self.g(work, "push", "-q", "origin", "main") + self.repos[name] = (origin, work) + os.makedirs(os.path.join(self.cloud, ".relmgr", ".github")) + os.symlink(HERE, os.path.join(self.cloud, ".relmgr", ".github", "release")) + bin_ = os.path.join(d, "bin") + os.mkdir(bin_) + with open(os.path.join(bin_, "gh"), "w") as fh: # other artifacts empty + fh.write(f"""#!/bin/sh +case "$*" in + *%2Fcloud%2Frelease/versions*) cat {d}/reg.jsonl 2>/dev/null;; + "pr list -R "*) echo "$GH_TOKEN $*" >> {d}/gh.log; [ -e {d}/prfail ] && exit 1 + cat "{d}/$(echo "$4" | tr / _).json" 2>/dev/null || echo "[]";; + *"/pulls"*) echo "$GH_TOKEN $*" >> {d}/gh.log; [ -e {d}/pullsfail ] && exit 1 + cat {d}/pulls.json 2>/dev/null || echo "[]";; +esac +true +""") + os.chmod(os.path.join(bin_, "gh"), 0o755) + self.path = f"{bin_}:{os.environ['PATH']}" + + cloud = property(lambda self: self.repos["cloud"][1]) + + def g(self, repo, *a): + return rm.git(repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a) + + def line( + self, x_y, *tags + ): # cloud and OSS release/X.Y; tags: cloud tags on the cloud tip, also in the registry + self.g(self.cloud, "push", "-q", "origin", f"main:refs/heads/release/{x_y}") + for i, t in enumerate(tags): + self.g(self.cloud, "tag", t, "main") + self.g(self.cloud, "push", "-q", "origin", t) + with open(os.path.join(self.d, "reg.jsonl"), "a") as fh: + fh.write( + f'{{"id": {len(t) * 10 + i}, "digest": "sha256:{t}", "pushed_at": "2026-09-01T00:00:00Z", ' + f'"tags": ["{t}"]}}\n' + ) + with contextlib.redirect_stderr(io.StringIO()): + oss.cut(self.repos["oss"][1], x_y) + + def run_step(self, script, **env): + self.g(self.cloud, "fetch", "-q", "--tags", "origin") + for name in ("output", "notes.md"): # a fresh run's files + with contextlib.suppress(FileNotFoundError): + os.remove(os.path.join(self.d, name)) + env = { + **os.environ, + "PATH": self.path, + "FORCE": "false", + "REASON": "", + "OSS_REPO": self.repos["oss"][1], + "OSS_TOKEN": "oss-token", + "GH_TOKEN": "cloud-token", + "RUNNER_TEMP": self.d, + "GITHUB_REPOSITORY": "Zipstack/unstract-cloud", + "GITHUB_STEP_SUMMARY": os.path.join(self.d, "summary"), + "GITHUB_OUTPUT": os.path.join(self.d, "output"), + **env, + } + return subprocess.run( + [sys.executable, "-"], + input=script, + cwd=self.cloud, + env=env, + capture_output=True, + text=True, + ) + + def read(self, name): + with open(os.path.join(self.d, name)) as fh: + return fh.read() + + def write(self, name, data): + with open(os.path.join(self.d, name), "w") as fh: + fh.write(data) + + +class RetireWorkflow(Workflow): + """Retire's inline script in each mode: the OSS line goes with the cloud one.""" + + def retire(self, **env): + r = self.run_step( + inline("retire.yaml", "Derive and retire"), + **{"ABANDON": "", "RETIRE_LINE": "", "NOW": "", "DRY_RUN": "false", **env}, + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def assert_gone(self, x_y): + c_origin, o_origin = self.repos["cloud"][0], self.repos["oss"][0] + self.assertNotIn(f"release/{x_y}", rm.git(c_origin, "branch", "-l", "release/*")) + self.assertNotIn(f"release/{x_y}", rm.git(o_origin, "branch", "-l", "release/*")) + self.assertEqual( + rm.git(o_origin, "tag", "-l", f"ee-v{x_y}.*"), "" + ) # no OSS GA: RC tags go + self.assertRegex( + rm.git(o_origin, "tag", "-l", f"retired/{x_y}-*"), + rf"^retired/{re.escape(x_y)}-[0-9a-f]{{7}}$", + ) + + def test_abandon(self): + self.line("1.7") + self.retire(ABANDON="release/1.7", REASON="cut by mistake") + self.assert_gone("1.7") + + def test_retire_line_no_ga(self): + self.line("1.7") + self.retire(RETIRE_LINE="release/1.7", REASON="not shipping") + self.assert_gone("1.7") + + def test_derived_list(self): + self.line("1.6", "v1.6.0-rc.1", "v1.6.0") + self.line( + "1.7", "v1.7.0-rc.1", "v1.7.0" + ) # 1.7 is cloud production: 1.6 is past end of support + self.retire() + self.assert_gone("1.6") + self.assertRegex( + rm.git( + self.repos["cloud"][0], + "tag", + "-l", + "retired/1.6-*", + "--format=%(refname:short) %(contents)", + ), + r"^retired/1\.6-[0-9a-f]{7} retired release/1\.6: past end of support", + ) + self.assertIn( + "release/1.7", rm.git(self.repos["oss"][0], "branch", "-l", "release/*") + ) + + def moved( + self, x_y + ): # an earlier run archived release/x_y, failed before the delete, then the line moved + origin = self.repos["cloud"][0] + tip = rm.git(origin, "rev-parse", f"release/{x_y}") + rm.git(origin, "tag", f"retired/{x_y}-{tip[:7]}", tip) + self.g(self.cloud, "commit", "-q", "--allow-empty", "-m", "late fix") + self.g(self.cloud, "push", "-q", "origin", f"HEAD:refs/heads/release/{x_y}") + return f"retired/{x_y}-{tip[:7]}" + + def refused(self, x_y, tag, **env): + origin = self.repos["cloud"][0] + r = self.run_step( + inline("retire.yaml", "Derive and retire"), + **{"ABANDON": "", "RETIRE_LINE": "", "NOW": "", "DRY_RUN": "false", **env}, + ) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn(f"after {tag} archived it", r.stdout) + self.assertEqual(rm.git(origin, "tag", "-l", "retired/*"), tag) + self.assertIn(f"release/{x_y}", rm.git(origin, "branch", "-l", "release/*")) + + def test_retire_line_refuses_a_line_moved_after_its_archive(self): + self.line("1.7") + self.refused( + "1.7", self.moved("1.7"), RETIRE_LINE="release/1.7", REASON="not shipping" + ) + + def test_derived_list_refuses_a_line_moved_after_its_archive(self): + self.line("1.6", "v1.6.0-rc.1", "v1.6.0") + tag = self.moved("1.6") + self.line("1.7", "v1.7.0-rc.1", "v1.7.0") + self.refused("1.6", tag) + + +class PendingPicksWorkflow(Workflow): + """Promote's Plan blocks on open backport-conflict PRs of the cloud line or the OSS line it pins, and on held + security/critical fixes, unless forced (recorded). A failed listing blocks too: forced, a failed conflict listing + passes and is recorded, a failed held listing still blocks. Take RC only warns. + """ + + def setUp(self): + super().setUp() + with open(os.path.join(self.cloud, "pyproject.toml"), "w") as fh: + fh.write('unstract-platform-version = "ee-v1.7.0-rc.1"\n') + self.g(self.cloud, "add", "pyproject.toml") + self.g(self.cloud, "commit", "-q", "-m", "pin") + self.g(self.cloud, "push", "-q", "origin", "main") + + def prs(self, slug, *bases): + self.write( + slug.replace("/", "_") + ".json", + json.dumps( + [ + { + "number": 40 + i, + "url": f"https://x/{i}", + "baseRefName": b, + "createdAt": "2026-10-01T00:00:00Z", + } + for i, b in enumerate(bases) + ] + ), + ) + + def promote(self, rc="v1.7.0-rc.1", **env): + return self.run_step(inline("promote.yaml", "Plan"), RC=rc, TARGET="cloud", **env) + + def blocked(self, r, msg): + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn(msg, r.stdout) + self.assertFalse(os.path.exists(os.path.join(self.d, "output"))) + + def forced(self, r): + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + return re.search(r"^forced=(.*)$", self.read("output"), re.M)[1] + + def test_promote_conflict_prs(self): + self.line("1.7", "v1.7.0-rc.1") + self.assertEqual(self.forced(self.promote()), "") + self.prs("Zipstack/unstract-cloud", "release/1.6") # another line's PR: no block + self.prs(oss.OSS, "release/1.6") + self.assertEqual(self.forced(self.promote()), "") + self.prs("Zipstack/unstract-cloud", "release/1.7") + self.blocked( + self.promote(), + "::error::Open backport conflict PR Zipstack/unstract-cloud#40 on release/1.7", + ) + self.prs("Zipstack/unstract-cloud") + self.prs(oss.OSS, "release/1.7") + self.blocked( + self.promote(), + f"::error::Open backport conflict PR {oss.OSS}#40 on release/1.7", + ) + self.assertIn(f"oss-token pr list -R {oss.OSS} ", self.read("gh.log")) + self.assertIn( + "cloud-token pr list -R Zipstack/unstract-cloud ", self.read("gh.log") + ) + self.assertEqual( + self.forced(self.promote(FORCE="true", REASON="ship it")), + f"ship it; open conflict PRs: {oss.OSS}#40", + ) + + def test_promote_fails_closed(self): + self.line("1.7", "v1.7.0-rc.1") + self.write("prfail", "") + self.blocked(self.promote(), "::error::cannot list open backport conflict PRs") + self.assertEqual( + self.forced(self.promote(FORCE="true", REASON="ship it")), + "ship it; open conflict PRs: unchecked: listing failed", + ) + + def test_promote_held_fixes(self): + self.line("1.7", "v1.7.0-rc.1", "v1.7.0") # cloud GA: the line is stabilized + self.g( + self.cloud, "commit", "-q", "--allow-empty", "-m", "fix: y" + ) # the next RC's commit + self.g(self.cloud, "tag", "v1.7.1-rc.1") + self.g( + self.cloud, + "push", + "-q", + "origin", + "main:refs/heads/release/1.7", + "v1.7.1-rc.1", + ) + with open(os.path.join(self.d, "reg.jsonl"), "a") as fh: + fh.write( + '{"id": 9, "digest": "sha256:rc", "pushed_at": "2026-09-01T00:00:00Z", "tags": ["v1.7.1-rc.1"]}\n' + ) + self.g( + self.cloud, "commit", "-q", "--allow-empty", "-m", "fix: x (#9)" + ) # on main only: held for 1.7 + self.g(self.cloud, "push", "-q", "origin", "main") + self.write( + "pulls.json", + json.dumps( + [{"number": 9, "title": "fix: x", "labels": [{"name": "fix/security"}]}] + ), + ) + r = self.promote("v1.7.1-rc.1") + self.blocked(r, "::error::pending picks") + self.assertRegex( + r.stdout, r"::error::[0-9a-f]{7} \(security, PR #9\) held for release/1\.7" + ) + self.assertNotIn( + f"oss-token api repos/{oss.OSS}/", self.read("gh.log") + ) # nothing on OSS main yet + self.assertEqual( + self.forced(self.promote("v1.7.1-rc.1", FORCE="true", REASON="next RC")), + "next RC", + ) + oss_work = self.repos["oss"][ + 1 + ] # a security fix on OSS main, not picked onto OSS release/1.7: held too + self.g(oss_work, "pull", "-q", "origin", "main") + self.g(oss_work, "commit", "-q", "--allow-empty", "-m", "fix: z (#9)") + self.g(oss_work, "push", "-q", "origin", "main") + self.g(oss_work, "fetch", "-q", "origin") + r = self.promote("v1.7.1-rc.1") + self.blocked(r, "::error::pending picks") + self.assertIn(f"oss-token api repos/{oss.OSS}/commits/", self.read("gh.log")) + self.assertEqual( + len( + re.findall( + r"::error::[0-9a-f]{7} \(security, PR #9\) held for release/1\.7", + r.stdout, + ) + ), + 1, + ) + self.assertRegex( + r.stdout, + rf"::error::[0-9a-f]{{7}} \(security, PR {re.escape(oss.OSS)}#9\) held for release/1\.7", + ) + take = self.run_step( + inline("take-rc-build.yaml", "Plan checks"), + BRANCH="release/1.7", + TARGETS="cloud", + ) # Take RC lists the OSS-held fix too, and never blocks on it + self.assertIn( + f"Held for the RM on {oss.OSS} `release/1.7`", + self.read("notes.md"), + take.stdout + take.stderr, + ) + self.assertIn(f"Held for the RM on {oss.OSS} `release/1.7`", self.read("summary")) + self.assertIn(f"backport.yaml -R {oss.OSS} -f commits=#9", self.read("summary")) + self.write( + "pullsfail", "" + ) # the held-fix listing fails: Take RC warns and goes on, Promote fails closed + take = self.run_step( + inline("take-rc-build.yaml", "Plan checks"), + BRANCH="release/1.7", + TARGETS="cloud", + ) + self.assertEqual(take.returncode, 0, take.stdout + take.stderr) + self.assertIn( + "::warning::⚠ Could not list held fixes on Zipstack/unstract-cloud", + take.stdout, + ) + self.blocked( + self.promote("v1.7.1-rc.1", FORCE="true", REASON="x"), + "::error::cannot list held security/critical fixes", + ) + self.assertIn("rerun Promote once GitHub answers", self.read("summary")) + + def test_take_rc_only_warns(self): + self.line("1.7") + self.prs(oss.OSS, "release/1.7") + script = inline("take-rc-build.yaml", "Plan checks") + self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + r = self.run_step(script, BRANCH="release/1.7", TARGETS="cloud") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn( + f"::warning::Open backport conflict PR {oss.OSS}#40 on release/1.7", r.stdout + ) + self.assertIn( + f"⚠ Open backport conflict PRs: [{oss.OSS}#40]", self.read("notes.md") + ) + self.write("prfail", "") + r = self.run_step(script, BRANCH="release/1.7", TARGETS="cloud") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("::warning::⚠ Could not list open backport conflict PRs", r.stdout) + + +class TakeRcVersion(Workflow): + def test_allocates_past_a_registry_rc(self): + # An earlier run pushed the cloud artifact of v1.7.0-rc.1, then its tag push failed and the tip moved. + self.line("1.7") + self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + self.write( + "reg.jsonl", + '{"id": 1, "digest": "sha256:a", "pushed_at": "2026-09-01T00:00:00Z", "tags": ["v1.7.0-rc.1"]}\n', + ) + r = self.run_step( + inline("take-rc-build.yaml", "Plan version and targets"), + BRANCH="release/1.7", + TARGETS="cloud", + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + self.read("output"), "tag=v1.7.0-rc.2\nbump=true\ntargets=cloud\n" + ) + + +class PublishWorkflow(Workflow): + """Publish's Plan reads the OSS pins of the RC and the previous stable; its Changelog step points the fetcher at + this repo and the OSS repo, and swaps in the bot token for the OSS call only. + """ + + def release( + self, pin, *tags + ): # a main commit pinning pin (None: no pyproject.toml), tagged; tags in the registry + path = os.path.join(self.cloud, "pyproject.toml") + if pin: + with open(path, "w") as fh: + fh.write(f'unstract-platform-version = "{pin}"\n') + self.g(self.cloud, "add", "pyproject.toml") + elif os.path.exists(path): + self.g(self.cloud, "rm", "-q", "pyproject.toml") + self.g(self.cloud, "commit", "-q", "--allow-empty", "-m", f"pin {pin}") + for t in tags: + self.g( + self.cloud, + "tag", + *( + ["-a", t, "-m", f"promote: {tags[0]} -> {t} (cloud)"] + if t != tags[0] + else [t] + ), + ) + with open(os.path.join(self.d, "reg.jsonl"), "a") as fh: + fh.write( + f'{{"id": {len(t)}, "digest": "sha256:{t}", "pushed_at": "2026-09-01T00:00:00Z", "tags": ["{t}"]}}\n' + ) + self.g(self.cloud, "push", "-q", "--tags", "origin", "main") + + def plan(self): + r = self.run_step( + inline("publish.yaml", "Plan"), + VERSION="v1.7.0", + TARGET="cloud", + OWNER_REPO="Zipstack/unstract-cloud", + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + return r.stdout, dict( + line.split("=", 1) for line in self.read("output").splitlines() + ) + + def test_oss_range_is_the_two_pins(self): + self.release("ee-v1.6.0", "v1.6.0") + self.release("ee-v1.7.0-rc.1", "v1.7.0-rc.1", "v1.7.0") + log, out = self.plan() + self.assertEqual((out["prev_oss"], out["oss"]), ("ee-v1.6.0", "ee-v1.7.0-rc.1")) + self.assertNotIn("::warning::no OSS pin", log) + + def test_a_side_with_no_pin_skips_oss(self): + self.release(None, "v1.6.0") + self.release("ee-v1.7.0-rc.1", "v1.7.0-rc.1", "v1.7.0") + log, out = self.plan() + self.assertIn("::warning::no OSS pin at v1.6.0: OSS changelog skipped", log) + self.assertEqual( + (out["prev_oss"], out["oss"]), ("ee-v1.7.0-rc.1", "ee-v1.7.0-rc.1") + ) # same pin: skipped + + def test_changelog_reads_both_repos_with_the_bot_token_for_oss_only(self): + # The real fetcher with a stand-in requests that logs each call and a main that calls it. + action = os.path.join(self.cloud, ".github", "actions", "update-changelogs") + os.makedirs(action) + os.symlink( + os.path.join(HERE, "..", "actions", "update-changelogs", "core"), + os.path.join(action, "core"), + ) + with open(os.path.join(action, "requests.py"), "w") as fh: + fh.write(f"""class exceptions: + RequestException = Exception +class R: + def raise_for_status(self): pass + def json(self): return {{"body": "notes"}} +def post(url, json, headers): + with open({os.path.join(self.d, "calls")!r}, "a") as f: + f.write(url.split("/repos/")[1].split("/releases")[0] + " " + headers["Authorization"] + "\\n") + return R() +""") + with open(os.path.join(action, "main.py"), "w") as fh: + fh.write("""import sys, core.changelog_fetcher as cf +def main(): + print("argv", sys.argv[1:]) + f = cf.ChangelogFetcher("workflow-token") + f._fetch_enterprise_changelog("v1.6.0", "v1.7.0-rc.1") + f._fetch_oss_changelog("ee-v1.6.0", "ee-v1.7.0-rc.1") + f._fetch_enterprise_changelog("v1.6.0", "v1.7.0-rc.1") +""") + r = self.run_step( + inline("publish.yaml", "Changelog"), + VERSION="v1.7.0", + RC="v1.7.0-rc.1", + PREV="v1.6.0", + OSS="ee-v1.7.0-rc.1", + PREV_OSS="ee-v1.6.0", + EXCLUDE="12,14", + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn( + "argv ['--release-version', 'v1.7.0', '--rc-tag', 'v1.7.0-rc.1', '--previous-stable', 'v1.6.0', " + "'--current-oss', 'ee-v1.7.0-rc.1', '--previous-oss', 'ee-v1.6.0', '--exclude-prs', '12,14']", + r.stdout, + ) + self.assertEqual( + self.read("calls").splitlines(), + [ + "Zipstack/unstract-cloud token workflow-token", + "Zipstack/unstract token oss-token", + "Zipstack/unstract-cloud token workflow-token", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py new file mode 100644 index 0000000000..dad4139fc6 --- /dev/null +++ b/.github/release/test_relstate.py @@ -0,0 +1,1085 @@ +"""Synthetic-repo tests for relstate (derive 1-11, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37). Run: python test_relstate.py""" + +import contextlib +import datetime as dt +import io +import os +import tempfile +import unittest + +os.environ["GITHUB_REPOSITORY"] = ( + "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets +) +import relmgr as rm +import relstate as rs + +# Isolate from the user's git config (signing, noprefix, hooks...). +os.environ.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") +os.environ.pop("GITHUB_OUTPUT", None) +NOW = dt.datetime(2026, 9, 29, tzinfo=dt.UTC) +CUT = {"branch": "release/1.5", "line": "1.5"} + + +def ago(days): + return (NOW - dt.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class Scenario(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.repo, self.reg, self.n = self._tmp.name, {}, 0 + rm.git(self.repo, "init", "-q", "-b", "main") + self.commit("base") + + def g(self, *args): + return rm.git( + self.repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args + ) + + def commit(self, msg, branch="main"): + if branch != "main": + self.g("checkout", "-q", branch) + self.g("commit", "-q", "--allow-empty", "-m", msg) + if branch != "main": + self.g("checkout", "-q", "main") + return self.g("rev-parse", branch) + + def cut(self, line): + self.g("branch", f"release/{line}", "main") + return self.commit(f"work on {line}", f"release/{line}") + + def push(self, artifact, tag, days=1, git_ref=None): + if git_ref: + self.g("tag", tag, git_ref) + self.n += 1 + self.reg.setdefault(artifact, {})[tag] = { + "digest": f"sha256:{self.n:064x}", + "pushed_at": ago(days), + } + + def state(self): + return rs.derive(self.repo, NOW, lambda a: self.reg.get(a, {})) + + def branch(self, st, name): + return next(b for b in st["branches"] if b["branch"] == name) + + def ga(self, artifact, tag): + return {"tag": tag, **self.reg[artifact][tag]} + + # Scenarios build on each other as the table does. + def s1(self): + self.cut("1.5") + self.push("cloud/release", "v1.5.0-rc.1", git_ref="release/1.5") + + def s2(self): + self.s1() + self.push("cloud/release", "v1.5.0", git_ref="release/1.5") + + def s3(self): + self.s2() + self.push("on-prem/release", "v1.5.0-rc.3", git_ref="release/1.5") + + def s4(self, chart_days=10): + self.s3() + self.push("on-prem/chart", "v1.5.1", days=chart_days) + + def s6(self, chart_days=10): + self.s4(chart_days) + self.cut("1.6") + self.push("cloud/release", "v1.6.0-rc.1", git_ref="release/1.6") + self.push("cloud/release", "v1.6.0", git_ref="release/1.6") + + def cloud_after_2(self, state): + ga = self.ga("cloud/release", "v1.5.0") + return {"state": state, "rcs": ["v1.5.0-rc.1"], "ga": ga, "promoted": ga} + + def onprem(self, state, chart=True): + return { + "state": state, + "rcs": ["v1.5.0-rc.3"], + "ga": self.ga("on-prem/chart", "v1.5.1") if chart else None, + "promoted": None, + } + + def test_1_cut_with_cloud_rc(self): + self.s1() + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "stabilizing", + "enabled": ["cloud"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": None, + "targets": { + "cloud": { + "state": "stabilizing", + "rcs": ["v1.5.0-rc.1"], + "ga": None, + "promoted": None, + } + }, + }, + ) + self.assertEqual(st["backport_targets"]["fix"], ["release/1.5"]) + self.assertEqual(st["supported"], {"cloud": ["release/1.5"], "on-prem": []}) + + def test_2_cloud_ga(self): + self.s2() + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "stabilized", + "enabled": ["cloud"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": 0, + "targets": {"cloud": self.cloud_after_2("stabilized")}, + }, + ) + self.assertEqual( + st["backport_targets"], + { + "fix": [], + "security_critical": [ + {"branch": "release/1.5", "needs_rm_confirmation": True} + ], + }, + ) + + def test_3_onprem_rc_reopens_branch(self): + self.s3() + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "stabilizing", + "enabled": ["cloud", "on-prem"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": 0, + "targets": { + "cloud": self.cloud_after_2("stabilized"), + "on-prem": self.onprem("stabilizing", chart=False), + }, + }, + ) + self.assertEqual(st["backport_targets"]["fix"], ["release/1.5"]) + + def test_4_both_stabilized(self): + self.s4() + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "stabilized", + "enabled": ["cloud", "on-prem"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": 0, + "targets": { + "cloud": self.cloud_after_2("stabilized"), + "on-prem": self.onprem("stabilized"), + }, + }, + ) + self.assertEqual(st["supported"]["on-prem"], ["release/1.5"]) + + def test_5_onprem_window_over(self): + self.s4(chart_days=200) + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "stabilized", + "enabled": ["cloud", "on-prem"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": 0, + "targets": { + "cloud": self.cloud_after_2("stabilized"), + "on-prem": self.onprem("past-eos"), + }, + }, + ) + self.assertEqual(st["supported"], {"cloud": ["release/1.5"], "on-prem": []}) + self.assertEqual(st["retire"], []) + + def test_6_newer_cloud_ga(self): + self.s6() + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "stabilized", + "enabled": ["cloud", "on-prem"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": 0, + "targets": { + "cloud": self.cloud_after_2("past-eos"), + "on-prem": self.onprem("stabilized"), + }, + }, + ) + self.assertEqual( + st["supported"], {"cloud": ["release/1.6"], "on-prem": ["release/1.5"]} + ) + + def test_7_retire(self): + self.s6(chart_days=200) + st = self.state() + self.assertEqual( + self.branch(st, "release/1.5"), + { + **CUT, + "state": "past-eos", + "enabled": ["cloud", "on-prem"], + "next_rc": None, + "tip_rc": "v1.5.0-rc.1", + "ahead_of_ga": 0, + "targets": { + "cloud": self.cloud_after_2("past-eos"), + "on-prem": self.onprem("past-eos"), + }, + }, + ) + self.assertEqual(st["retire"], ["release/1.5"]) + self.assertEqual(st["supported"], {"cloud": ["release/1.6"], "on-prem": []}) + + def test_7b_onprem_newest_six(self): + for y in range(1, 9): # 1.1 .. 1.8, each with an on-prem chart inside the window + self.cut(f"1.{y}") + self.push("on-prem/release", f"v1.{y}.0-rc.1", git_ref=f"release/1.{y}") + self.push("on-prem/chart", f"v1.{y}.0", days=10) + self.cut("1.9") # no on-prem GA: does not count toward the six + self.push("on-prem/release", "v1.9.0-rc.1", git_ref="release/1.9") + st = self.state() + self.assertEqual( + st["supported"]["on-prem"], [f"release/1.{y}" for y in range(9, 2, -1)] + ) + self.assertEqual(st["retire"], ["release/1.2", "release/1.1"]) + + def test_8_cut_no_artifacts(self): + self.s4() + self.cut("1.7") + st = self.state() + self.assertEqual( + self.branch(st, "release/1.7"), + { + "branch": "release/1.7", + "line": "1.7", + "state": "stabilizing", + "enabled": [], + "next_rc": "v1.7.0-rc.1", + "tip_rc": None, + "ahead_of_ga": None, + "targets": {}, + }, + ) + self.assertTrue(all("release/1.7" not in v for v in st["supported"].values())) + self.assertIn("release/1.7", st["backport_targets"]["fix"]) + + def test_9_tag_not_on_branch(self): + self.s1() + self.push("on-prem/release", "v1.5.0-rc.9", git_ref=self.commit("main-only")) + st = self.state() + self.assertEqual(self.branch(st, "release/1.5")["enabled"], ["cloud"]) + self.assertEqual( + st["anomalies"], + [ + { + "target": "on-prem", + "artifact": "on-prem/release", + "tag": "v1.5.0-rc.9", + "branch": "release/1.5", + "reason": "commit not on branch", + } + ], + ) + + def test_10_registry_failure(self): + self.s1() + + def boom(artifact): + raise RuntimeError("HTTP 403") + + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = rs.main( + ["--repo", self.repo, "--now", "2026-09-29T00:00:00Z"], registry=boom + ) + self.assertNotEqual(code, 0) + self.assertEqual(out.getvalue(), "") + self.assertIn("HTTP 403", err.getvalue()) + + def test_11_next_minor_ignores_legacy(self): + self.cut("1.5") + self.cut("1.6") + self.g("branch", "release/v0.148.1001", "main") + st = self.state() + self.assertEqual(st["next_minor"], "1.7") + self.assertEqual( + [b["branch"] for b in st["branches"]], ["release/1.6", "release/1.5"] + ) + + def test_13_retired_lines_are_history_not_anomalies(self): + self.s4() # only release/1.5 is live + retired = self.commit("1.6 work") + self.push( + "cloud/release", "v1.6.0-rc.1", git_ref=retired + ) # promoted, published, then retired: no branch + self.push("cloud/release", "v1.6.0", git_ref=retired) + self.push("on-prem/release", "v1.6.0") + self.push("on-prem/chart", "v1.6.0", days=400) + self.push( + "cloud/release", "v1.9.0-rc.1", git_ref=self.commit("never promoted") + ) # still an anomaly + st = self.state() + self.assertEqual( + st["anomalies"], + [ + { + "target": "cloud", + "artifact": "cloud/release", + "tag": "v1.9.0-rc.1", + "branch": "release/1.9", + "reason": "no branch", + } + ], + ) + self.assertEqual([b["branch"] for b in st["branches"]], ["release/1.5"]) + + def test_14_next_rc_moves_after_tip_is_built(self): + self.s2() + self.commit("pick", "release/1.5") + self.assertEqual( + { + k: self.branch(self.state(), "release/1.5")[k] + for k in ("next_rc", "tip_rc") + }, + {"next_rc": "v1.5.1-rc.1", "tip_rc": None}, + ) + + def test_15_summary(self): + self.s6() + out = rs.summary(self.state()) + self.assertIn( + "| release/1.6 | stabilized | stabilized (GA v1.6.0) | - | v1.6.0-rc.1 | - | 0 |", + out, + ) + self.assertIn( + "| release/1.5 | stabilized | past-eos | stabilized (GA v1.5.1) | v1.5.0-rc.1 | - | 0 |", + out, + ) + self.assertIn("next_minor: 1.7", out) + + def test_19_ahead_of_ga(self): + self.s2() + self.g("tag", "v1.5.1", "release/1.5") # latest stable tag counts, not the first + self.commit("pick 1", "release/1.5") + self.commit("pick 2", "release/1.5") + self.cut("1.6") # no stable tag yet + st = self.state() + self.assertEqual( + [self.branch(st, b)["ahead_of_ga"] for b in ("release/1.5", "release/1.6")], + [2, None], + ) + self.assertIn("| v1.5.2-rc.1 | 2 |", rs.summary(st)) + + def test_21_rbo_line_in_squash_message(self): + self.cut("1.5") + squash = ( + "UN-37 [MISC] pin redis (#9)\n\n* chore: pin redis\n\nwhy it is pinned\n\n" + "* Release-branch-only: UN-37\n\n---------\n\nCo-authored-by: x " + ) + cases = { + squash: "release-branch-only", # COMMIT_MESSAGES squash: the bot's subject-only commit as `* ...` + "fix: pin\n\nbody\n\nRelease-branch-only: UN-37": "release-branch-only", # a trailer + "fix: pin\n\nsee the Release-branch-only: UN-37 note": "invalid", # mid-line is not the line + "fix: pin\n\n** Release-branch-only: UN-37": "invalid", + } + for msg, kind in cases.items(): + sha = self.commit(msg, "release/1.5") + self.assertEqual(rm.classify_commit(self.repo, sha), kind, msg) + self.assertEqual(rm.RBO_RE.findall(squash), ["UN-37"]) + + def test_22_promote_blockers(self): + """Promote order per target: only an older line with the target enabled and not GA for it blocks.""" + msg = "release/{} is not GA for {} yet: promote and publish it first, retire it, or force with a reason" + self.s3() # 1.5: cloud GA, on-prem RC only + self.cut("1.6") + self.push("cloud/release", "v1.6.0-rc.1", git_ref="release/1.6") + self.cut("1.7") # nothing built yet + st = self.state() + self.assertEqual( + rs.promote_blockers(st, "release/1.5", "cloud"), [] + ) # no older line + # (a) 1.5 is GA for cloud only: 1.6 cloud passes, 1.6 on-prem waits for 1.5 + self.assertEqual(rs.promote_blockers(st, "release/1.6", "cloud"), []) + self.assertEqual( + rs.promote_blockers(st, "release/1.6", "on-prem"), + [msg.format("1.5", "on-prem")], + ) + # (b) on-prem is not enabled on 1.6, so 1.6 never blocks on-prem + # (c) 1.7, nothing built, waits per target only for older lines with that target enabled and not GA + self.assertEqual( + rs.promote_blockers(st, "release/1.7", "on-prem"), + [msg.format("1.5", "on-prem")], + ) + self.assertEqual( + rs.promote_blockers(st, "release/1.7", "cloud"), [msg.format("1.6", "cloud")] + ) + # and 1.7 itself, with nothing enabled, blocks nothing newer beyond those two + self.assertEqual( + rs.promote_blockers(st, "release/1.8", "cloud"), [msg.format("1.6", "cloud")] + ) + # (d) 1.5 publishes on-prem (chart push): 1.6 on-prem is no longer blocked + self.push("on-prem/chart", "v1.5.1") + self.assertEqual(rs.promote_blockers(self.state(), "release/1.6", "on-prem"), []) + # (e) a retired older line (branch gone) never blocks + self.g("branch", "-D", "release/1.6") + self.assertEqual(rs.promote_blockers(self.state(), "release/1.7", "cloud"), []) + + def test_23_enable_blockers(self): + """A target is enabled only on a line newer than every line already GA for it.""" + msg = "release/{} is older than release/{}, already GA for {}: enable {} on a newer line instead" + self.cut("1.4") + self.push("cloud/release", "v1.4.0-rc.1", git_ref="release/1.4") + self.s6() # 1.5: cloud GA, on-prem GA (chart v1.5.1); 1.6: cloud GA, no on-prem + st = self.state() + # (a) an older line enabling on-prem behind a line GA for on-prem is refused + self.assertEqual( + rs.enable_blockers(st, "release/1.4", "on-prem"), + [msg.format("1.4", "1.5", "on-prem", "on-prem")], + ) + # (b) the newest line may enable it + self.assertEqual(rs.enable_blockers(st, "release/1.6", "on-prem"), []) + # (c) a target already enabled on the older line is not affected: a patch RC still builds + self.assertEqual(rs.enable_blockers(st, "release/1.4", "cloud"), []) + self.assertEqual(rs.enable_blockers(st, "release/1.5", "on-prem"), []) + + def test_23b_no_order_block_on_a_ga_line(self): + """(d) Invariant: in a state reached only by moves the enable rule and the promote order allow, a patch on a + line already GA for a target is never order-blocked. Without the enable rule it can be. + """ + rc, ga = ( + {"cloud": ("cloud/release", ".0"), "on-prem": ("on-prem/release", ".1")}, + {"cloud": "cloud/release", "on-prem": "on-prem/chart"}, + ) + refused = [] + + def enable(line, t): + if rs.enable_blockers(self.state(), f"release/{line}", t): + return refused.append(("enable", line, t)) + self.push( + rc[t][0], f"v{line}{rc[t][1]}-rc.{self.n}", git_ref=f"release/{line}" + ) # a later call is a patch RC + + def go_ga(line, t): + st = self.state() + if t not in self.branch(st, f"release/{line}")[ + "targets" + ] or rs.promote_blockers(st, f"release/{line}", t): + return refused.append(("ga", line, t)) + self.push( + ga[t], + f"v{line}{rc[t][1]}", + git_ref=f"release/{line}" if t == "cloud" else None, + ) + + def check(): + st = self.state() + for b in st["branches"]: + for t, v in b["targets"].items(): + if v["ga"]: + self.assertEqual( + rs.promote_blockers(st, b["branch"], t), [], (b["branch"], t) + ) + + self.cut("1.4") + self.cut("1.5") + moves = [ + (enable, "1.5", "cloud"), + (go_ga, "1.5", "cloud"), + (enable, "1.5", "on-prem"), + (self.cut, "1.6"), + (enable, "1.6", "cloud"), + (enable, "1.6", "on-prem"), + (go_ga, "1.6", "on-prem"), + (go_ga, "1.6", "cloud"), + (go_ga, "1.5", "on-prem"), + (enable, "1.4", "on-prem"), + (enable, "1.4", "cloud"), + (go_ga, "1.6", "on-prem"), + (self.cut, "1.7"), + (enable, "1.7", "on-prem"), + (enable, "1.5", "on-prem"), + ] + for f, *args in moves: + f(*args) + check() + self.assertEqual( + refused, + [ + ("ga", "1.6", "on-prem"), + ("enable", "1.4", "on-prem"), + ("enable", "1.4", "cloud"), + ], + ) + # enabling on-prem on 1.4 behind GA lines anyway order-blocks every newer GA line's patch + self.push("on-prem/release", "v1.4.1-rc.1", git_ref="release/1.4") + self.assertTrue(rs.promote_blockers(self.state(), "release/1.6", "on-prem")) + + def chart(self, ver, branch, msg, email="dev@example.com"): + self.g("checkout", "-q", branch) + os.makedirs(os.path.join(self.repo, os.path.dirname(rm.CHART)), exist_ok=True) + with open(os.path.join(self.repo, rm.CHART), "w") as f: + f.write(f"name: x\nversion: {ver}\n") + self.g("add", rm.CHART) + rm.git( + self.repo, + "-c", + "user.name=bot", + "-c", + f"user.email={email}", + "commit", + "-q", + "-m", + msg, + ) + self.g("checkout", "-q", "main") + + def test_24_already_bumped(self): + self.cut("1.5") + self.chart("1.5.0-rc.1", "release/1.5", "release: chart 1.5.0-rc.1", rm.BOT_EMAIL) + self.assertTrue(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.1")) + self.assertFalse( + rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.2") + ) # chart not at the version + self.commit( + "fix after the bump", "release/1.5" + ) # chart still at the version, tip is a normal commit + self.assertFalse(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.1")) + self.chart( + "1.5.0-rc.2", "release/1.5", "release: chart 1.5.0-rc.2" + ) # right subject, not the bot + self.assertFalse(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.2")) + + def test_25_promoted_from_build_artifact(self): + self.s3() + self.push( + "on-prem/release", "v1.5.1", git_ref="release/1.5" + ) # Promote on-prem, chart not published + onprem = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] + self.assertEqual( + onprem, + { + "state": "stabilizing", + "rcs": ["v1.5.0-rc.3"], + "ga": None, + "promoted": self.ga("on-prem/release", "v1.5.1"), + }, + ) + self.push( + "on-prem/chart", "v1.5.1" + ) # Publish: GA and state move, promoted does not + onprem = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] + self.assertEqual( + (onprem["state"], onprem["ga"], onprem["promoted"]), + ( + "stabilized", + self.ga("on-prem/chart", "v1.5.1"), + self.ga("on-prem/release", "v1.5.1"), + ), + ) + + def test_20_tip_rc_slack(self): + self.s1() + self.assertNotIn( + "tip_rc_slack", self.branch(self.state(), "release/1.5") + ) # lightweight tag: no key + url = "https://example.slack.com/archives/C0123ABC/p1759900000000100" + self.g( + "tag", + "-f", + "-a", + "v1.5.0-rc.1", + "-m", + "rc: built for x", + "-m", + f"Slack: {url}", + "release/1.5", + ) + st = self.state() + self.assertEqual(self.branch(st, "release/1.5")["tip_rc_slack"], url) + self.assertIn(f"| v1.5.0-rc.1 ([Slack]({url})) |", rs.summary(st)) + + def test_12_next_minor_cutover_fallback(self): + self.g("tag", "v0.181.0") + self.g("tag", "v0.181.1001") + self.g("tag", "v0.182.0-rc.1") # RC tags do not count + self.g("branch", "side", "main") + self.commit("off-main", "side") + self.g("tag", "v0.190.0", "side") # stable, but not reachable from main + self.g( + "branch", "release/v0.148.1001", "main" + ) # legacy head: still the cutover case + st = self.state() + self.assertEqual((st["branches"], st["next_minor"]), ([], "0.182")) + self.cut("0.182") + self.assertEqual(self.state()["next_minor"], "0.183") + + def built_by_mistake(self): + """release/1.9: cut, one bot commit, built once as v1.9.0-rc.1 on cloud and on-prem.""" + self.g("checkout", "-q", "-b", "release/1.9") + rm.git( + self.repo, + "-c", + "user.name=bot", + "-c", + f"user.email={rm.BOT_EMAIL}", + "commit", + "-q", + "--allow-empty", + "-m", + "pin", + ) + self.g("checkout", "-q", "main") + self.push("cloud/release", "v1.9.0-rc.1", git_ref="release/1.9") + self.push("on-prem/release", "v1.9.0-rc.1") + self.push( + "cloud/release", "v1.5.0" + ) # other versions keep the cloud package non-empty + + def test_16_abandon_plan_built_by_mistake(self): + self.built_by_mistake() + self.push("cloud/release", "v1.9.0-rc.2") + self.reg["cloud/release"]["edge"] = self.reg["cloud/release"][ + "v1.9.0-rc.2" + ] # mixed tags: kept + plan = rs.abandon_plan(self.repo, "release/1.9", lambda a: self.reg.get(a, {})) + rc1 = self.reg["cloud/release"]["v1.9.0-rc.1"]["digest"] + self.assertEqual( + plan, + { + "ga": [], + "used": [], + "rc_tags": ["v1.9.0-rc.1"], + "versions": { + "cloud/release": [ + {"id": None, "digest": rc1, "tags": ["v1.9.0-rc.1"]} + ] + }, + "kept": ["on-prem/release:v1.9.0-rc.1"], + }, + ) # last version of on-prem/release + self.g("branch", "release/1.8", "main") # never built: nothing to delete + self.assertEqual( + rs.abandon_plan(self.repo, "release/1.8", lambda a: {})["versions"], {} + ) + + def test_17_abandon_plan_refusals(self): + self.built_by_mistake() + plan = lambda: rs.abandon_plan( + self.repo, "release/1.9", lambda a: self.reg.get(a, {}) + ) + self.push("on-prem/chart", "v1.9.0") + self.assertEqual(plan()["ga"], ["on-prem/chart:v1.9.0"]) + self.g("tag", "v1.9.0", "release/1.9") + self.assertEqual(plan()["ga"], ["v1.9.0", "on-prem/chart:v1.9.0"]) + human = self.commit("human RBO-less work", "release/1.9") + self.assertEqual(plan()["used"], [human[:12]]) + + def test_26_oss_repo_config(self): + # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), + # supported while its branch exists. Plain v* tags are community releases from main and never count. + self.addCleanup(rs.use, rs.PKG_PREFIX) + rs.use("unstract") + self.state = lambda: rs.derive( + self.repo, NOW, lambda a: self.fail(f"registry read of {a}") + ) + self.g("tag", "v1.9.0") # a community release on main + for line in ("1.5", "1.6"): + self.cut(line) + self.g("tag", f"ee-v{line}.0-rc.1", f"release/{line}") + self.g("tag", f"ee-v{line}.0", f"release/{line}") + self.g("tag", f"v{line}.7", f"release/{line}") # never the line's GA + self.cut("1.7") + self.g("tag", "ee-v1.7.0-rc.1", "release/1.7") + st = self.state() + self.assertEqual( + [(b["branch"], b["state"]) for b in st["branches"]], + [ + ("release/1.7", "stabilizing"), + ("release/1.6", "stabilized"), + ("release/1.5", "stabilized"), + ], + ) + self.assertEqual( + (st["retire"], st["backport_targets"]["fix"], st["anomalies"]), + ([], ["release/1.7"], []), + ) + self.assertEqual( + st["next_minor"], "1.8" + ) # the community v1.9.0 on main is not this repo's minor + self.assertEqual( + self.branch(st, "release/1.6")["targets"]["oss"]["ga"]["tag"], "ee-v1.6.0" + ) + self.assertIn( + "| branch | state | oss | tip_rc | next_rc | ahead_of_ga |", rs.summary(st) + ) + self.assertIn( + "| release/1.6 | stabilized | stabilized (GA ee-v1.6.0) | ee-v1.6.0-rc.1 | - | 0 |", + rs.summary(st), + ) + self.commit("fix on 1.6", "release/1.6") + self.assertEqual( + self.branch(self.state(), "release/1.6")["next_rc"], "ee-v1.6.1-rc.1" + ) + # No promote order for OSS: 1.8 promotes while 1.7 is not GA; cloud's order would block it. + self.cut("1.8") + self.g("tag", "ee-v1.8.0-rc.1", "release/1.8") + st = self.state() + self.assertEqual(rs.promote_blockers(st, "release/1.8", "oss"), []) + rs.KEPT = set() + self.assertEqual( + rs.promote_blockers(st, "release/1.8", "oss"), + [ + "release/1.7 is not GA for oss yet: promote and publish it first, retire it, or force with a reason" + ], + ) + + def test_27_next_minor_past_stable_tag_on_main(self): + # A minor already shipped from main (tags before the first line) is never cut again. + self.g("tag", "v1.5.14") + self.commit("more main work") + self.g("tag", "v1.6.0") + self.g("branch", "release/1.5", "v1.5.14") + self.assertEqual(self.state()["next_minor"], "1.7") + + def test_28_already_bumped_no_chart(self): + self.cut("1.5") + + def bump(msg): + rm.git( + self.repo, + "-c", + "user.name=bot", + "-c", + f"user.email={rm.BOT_EMAIL}", + "commit", + "-q", + "--allow-empty", + "-m", + msg, + ) + + self.g("checkout", "-q", "release/1.5") + bump("release: 1.5.0-rc.1") + self.assertTrue(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.1")) + self.assertFalse(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.2")) + bump("release: chart 1.5.0-rc.2") # the chart subject needs a chart + self.assertFalse(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.2")) + + def test_29_retire_line(self): + refusal = lambda force=False: rs.retire_line_refusal( + self.state(), "release/1.5", force + ) + self.s3() # cloud GA newest (production), on-prem RCs only + self.assertEqual( + refusal(True), + "cloud is production (newest cloud GA); it retires once a newer line is promoted", + ) + self.cut("1.6") + self.push("cloud/release", "v1.6.0-rc.1", git_ref="release/1.6") + self.push("cloud/release", "v1.6.0", git_ref="release/1.6") + self.assertIsNone(refusal()) # cloud past-eos, on-prem never promoted: may go + self.push("on-prem/chart", "v1.5.1") # on-prem chart in its window + self.assertEqual( + refusal(), + "on-prem still supported (stabilized); force to retire it inside the window", + ) + self.assertIsNone(refusal(True)) + self.assertEqual( + rs.retire_line_refusal(self.state(), "release/9.9"), "no such branch" + ) + # The newest line retired before any GA (its RCs deleted): the next cut reuses its minor. + self.cut("1.7") + tip = self.g("rev-parse", "release/1.7") + self.g("tag", f"retired/1.7-{tip[:7]}", tip) + self.g("branch", "-D", "release/1.7") + self.assertEqual(self.state()["next_minor"], "1.7") + # Retired after a GA (forced on-prem, say): the minor shipped and is skipped. + self.g("tag", "v1.7.0", tip) + self.assertEqual(self.state()["next_minor"], "1.8") + + def pick_of(self, fix): # a non-empty pick of fix on the checked-out branch + with open(os.path.join(self.repo, "p.txt"), "a") as f: + f.write("x\n") + self.g("add", "p.txt") + self.g("commit", "-q", "-m", f"fix on main\n\n(cherry picked from commit {fix})") + + def test_30_rc_version_skips_registry_rcs(self): + # An artifact pushed by a run whose tag push failed: its RC is never allocated again once the tip moves. + self.cut("1.5") + taken = {"v1.5.0-rc.1", "v1.5.0-rc.2", "v1.4.0-rc.9", "v1.5.1-rc.7", "edge"} + self.assertEqual(rm.rc_version(self.repo, "release/1.5"), "v1.5.0-rc.1") + self.assertEqual( + rm.rc_version(self.repo, "release/1.5", taken=taken), "v1.5.0-rc.3" + ) + # A tip that is already the bot's bump for a pushed RC resumes that run, and so does a fresh bump. + for n in (2, 3): + self.chart( + f"1.5.0-rc.{n}", + "release/1.5", + f"release: chart 1.5.0-rc.{n}", + rm.BOT_EMAIL, + ) + self.assertEqual( + rm.rc_version(self.repo, "release/1.5", taken=taken), f"v1.5.0-rc.{n}" + ) + self.chart( + "1.5.0-rc.2", "release/1.5", "release: chart 1.5.0-rc.2" + ) # not the bot: no resume + self.assertEqual( + rm.rc_version(self.repo, "release/1.5", taken=taken), "v1.5.0-rc.3" + ) + # relstate allocates past every target's registry RCs. + self.push("cloud/release", "v1.5.0-rc.3") + self.push("on-prem/release", "v1.5.0-rc.4") + self.assertEqual( + self.branch(self.state(), "release/1.5")["next_rc"], "v1.5.0-rc.5" + ) + + def test_31_reused_minor_skips_its_kept_rc(self): + # release/1.7 built once, then abandoned: its git RC tag deleted, its registry RC kept (the package's last + # version). The minor is cut again, and its first RC is past the kept one. + self.cut("1.6") + tip = self.cut("1.7") + self.push("on-prem/release", "v1.7.0-rc.1") + self.g("tag", f"retired/1.7-{tip[:7]}", tip) + self.g("branch", "-D", "release/1.7") + self.assertEqual(self.state()["next_minor"], "1.7") + self.cut("1.7") + self.assertEqual( + self.branch(self.state(), "release/1.7")["next_rc"], "v1.7.0-rc.2" + ) + + def test_32_shallow_clone_refused(self): + clone = os.path.join(self._tmp.name, "shallow") + self.commit("second") + rm.git( + self._tmp.name, "clone", "-q", "--depth", "1", f"file://{self.repo}", clone + ) + with self.assertRaisesRegex(RuntimeError, "is a shallow clone"): + rm.git(clone, "log", "-1") + self.assertNotIn(clone, rm._FULL) + + def test_33_conflict_markers_git_error(self): + with open(os.path.join(self.repo, "f.txt"), "w") as f: + f.write("a\n<<<<<<< ours\nb\n=======\nc\n>>>>>>> theirs\n") + self.g("add", "f.txt") + self.g("commit", "-q", "-m", "markers") + self.assertEqual(rm.conflict_markers(self.repo), ["f.txt"]) + with self.assertRaisesRegex(RuntimeError, r"git grep \(exit 128\)"): + rm.conflict_markers(self.repo, "no-such-rev") + + def test_34_reverted_pick_only_on_its_own_branch(self): + fix = self.commit("fix on main") + self.cut("1.5") + self.cut("1.6") + self.g("checkout", "-q", "release/1.6") + self.pick_of(fix) + pick = self.g("rev-parse", "HEAD") + self.g("checkout", "-q", "main") + revert = f"Revert pick\n\nThis reverts commit {pick}." + self.assertEqual( + rm.reverted_pick(self.repo, self.commit(revert, "release/1.6")), pick + ) + other = self.commit( + revert, "release/1.5" + ) # the pick is on another line, not this one + self.assertIsNone(rm.reverted_pick(self.repo, other)) + self.assertEqual(rm.classify_commit(self.repo, other), "invalid") + self.assertIsNone( + rm.reverted_pick( + self.repo, + self.commit(f"Revert\n\nThis reverts commit {fix}.", "release/1.5"), + ) + ) + + def test_35_ci_commits(self): + fix = self.commit("fix on main") + self.cut("1.5") # one invalid commit, `work on 1.5` + before = self.g("rev-parse", "release/1.5") + self.g("checkout", "-q", "-b", "head", "release/1.5") + self.pick_of(fix) + pick = self.g("rev-parse", "HEAD") + self.g("commit", "-q", "--allow-empty", "-m", "resolution") + human = self.g("rev-parse", "HEAD") + work = self.g("rev-parse", "release/1.5") + pr = "PR into release/1.5" + ci = lambda event, **kw: rm.ci_commits( + self.repo, "release/1.5", event, main="main", **kw + ) + cases = [ + ( + ("pull_request",), + {"head_ref": "fix/x"}, + [(pick, "cherry-pick"), (human, "invalid")], + pr, + ), + ( + ("pull_request",), + {"head_ref": "fix/x", "labelled": True}, + [(pick, "cherry-pick"), (human, "release-branch-only (label)")], + pr, + ), + ( + ("pull_request",), + {"head_ref": f"backport/1.5/{fix[:7]}"}, + [(pick, "cherry-pick")], + pr, + ), + ( + ("pull_request",), + {"head_ref": "backport/anything"}, + [(pick, "cherry-pick"), (human, "invalid")], + pr, + ), + ( + ("push",), + {"before": before, "labelled": True}, + [(pick, "cherry-pick"), (human, "invalid")], + "push to release/1.5", + ), + (("push",), {"before": "0" * 40}, [(work, "invalid")], "release/1.5"), + ( + ("push",), + {"before": "f" * 40}, + [(work, "invalid")], + "release/1.5", + ), # a force push: old tip gone + (("push",), {}, [(work, "invalid")], "release/1.5"), + ] + for args, kw, rows, where in cases: + self.assertEqual(ci(*args, **kw), (rows, where), (args, kw)) + self.g("checkout", "-q", "main") + other = self.commit("other fix on main") + self.g("checkout", "-q", "head") + self.assertEqual( + ci("pull_request", head_ref=f"backport/1.5/{other[:7]}")[0], + [(pick, "invalid")], + ) # not its fix + self.g("checkout", "-q", "-B", "two", "release/1.5") + self.commit( + f"fix on main\n\n(cherry picked from commit {fix})\n(cherry picked from commit {other})" + ) + self.assertEqual( + ci("pull_request", head_ref=f"backport/1.5/{fix[:7]}")[0][0][1], "invalid" + ) # names a second fix + self.g("checkout", "-q", "head") + with open(os.path.join(self.repo, "f.txt"), "w") as f: + f.write(">>>>>>> theirs\n") + self.g("add", "f.txt") + self.g("commit", "-q", "-m", "left a marker") + with self.assertRaisesRegex(ValueError, "conflict markers in f.txt"): + ci("pull_request", head_ref=f"backport/1.5/{fix[:7]}") + self.assertEqual( + len(ci("pull_request", head_ref="fix/x")[0]), 3 + ) # only a backport PR's tree is checked + self.g("checkout", "-q", "main") + + def test_36_is_backport(self): + for ref, want in ( + ("backport/1.5/abcdef0", True), + ("backport/0.192/1234567", True), + ("backport/1.5/abcdef01", False), + ("backport/1.5/ABCDEF0", False), + ("backport/x", False), + ("evil/backport/1.5/abcdef0", False), + ("backport/1.5/abcdef0/x", False), + ("", False), + ): + self.assertEqual(rm.is_backport(ref), want, ref) + + def test_37_unknown_repo_config(self): + with self.assertRaisesRegex( + RuntimeError, "relstate has no config for repo 'nope'" + ): + rs.use("nope") + self.assertEqual(rs.PREFIX, "") # the loaded config is untouched + + +class PrFixClass(unittest.TestCase): + """Lint PR / Backport class rule: title type plus fix/ labels.""" + + def test_18_pr_fix_class(self): + c = rm.pr_fix_class + self.assertEqual(c("UN-1 [FIX] x"), "fix") + self.assertIsNone(c("UN-1 [MISC] x")) + self.assertEqual(c("UN-1 [MISC] x", ["fix/needs-backport"]), "fix") + self.assertEqual( + c("UN-1 [FIX] x", ["fix/needs-backport"]), "fix" + ) # no-op on a fix title + self.assertEqual( + c("UN-1 [MISC] x", ["fix/security", "fix/needs-backport"]), "security" + ) + self.assertEqual(c("UN-1 [FEAT] x", ["fix/critical", "bug"]), "critical") + for labels in ( + ["fix/typo"], + ["fix-class/security"], + ["Fix/security"], + ["fixclass"], + ["fix/security", "fix/critical"], + ): + with self.assertRaises(ValueError, msg=labels) as e: + c("UN-1 [FIX] x", labels) + if labels != ["fix/security", "fix/critical"]: + self.assertIn( + "fix/security, fix/critical, fix/needs-backport", str(e.exception) + ) + + def test_18b_revert_takes_inner_class(self): + c = rm.pr_fix_class + self.assertEqual(c('Revert "UN-1 [FIX] x"'), "fix") + self.assertIsNone(c('Revert "UN-1 [MISC] x"')) + self.assertEqual(c('Revert "Revert "UN-1 [FIX] x""'), "fix") + for title in ('Revert "fix x"', "Revert UN-1 [FIX] x", 'revert "UN-1 [FIX] x"'): + with self.assertRaises(ValueError, msg=title): + c(title) + with self.assertRaises(ValueError): + c("no prefix") + self.assertEqual(c("no prefix", ["fix/needs-backport"]), "fix") + + def test_18c_ticketless_title_same_class(self): + c = rm.pr_fix_class + self.assertEqual(c("[FIX] x"), "fix") + self.assertEqual(c("[HOT-FIX] x"), "fix") + self.assertIsNone(c("[MISC] x")) + self.assertEqual(c("[MISC] x", ["fix/needs-backport"]), "fix") + self.assertEqual(c("[FEAT] x", ["fix/security"]), "security") + self.assertEqual(c('Revert "[FIX] x"'), "fix") + for title in ("[fix] x", "[FIX]x", " [FIX] x", "UN-1[FIX] x", "un-1 [FIX] x"): + with self.assertRaises(ValueError, msg=title): + c(title) + + def test_18d_rbo_key_ticket_else_pr(self): + self.assertEqual(rm.rbo_key("UN-1 [FIX] x", "45"), "UN-1") + self.assertEqual(rm.rbo_key("[FIX] x", "45"), "#45") + self.assertEqual(rm.rbo_key("no prefix", "45"), "#45") + self.assertEqual(rm.RBO_RE.findall("Release-branch-only: #45"), ["#45"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml new file mode 100644 index 0000000000..3d60a62542 --- /dev/null +++ b/.github/workflows/lint-pr.yaml @@ -0,0 +1,69 @@ +name: Lint PR + +# Every PR into main or a release branch carries a recognised `[TYPE]` title prefix, the ticket before it optional +# (`UN-123 [FIX] ...` or `[FIX] ...`, the same class either way), and its labels are +# at most one of fix/security, fix/critical plus an optional fix/needs-backport. +# Release branches too, because the release changelog is built from their PR titles. A branch cut before this file +# existed runs no check. Not a required check until a ruleset makes it one. +# The backport class is only printed here; Backport recomputes it with pr_fix_class at pick time. +# Label `release-branch-only` marks a change that lives only on its release branch: refused on main. On a release +# branch, Release branch CI's bot commit writes `Release-branch-only: ` (or `#` without a ticket), which +# Squash and merge keeps in the message. +on: + pull_request: + branches: [main, 'release/**'] + types: [opened, edited, reopened, synchronize, labeled, unlabeled] + +permissions: + contents: read + +jobs: + lint-pr: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # The rule comes from main, so a PR's edit to relmgr cannot pass its own check. That guards relmgr only: a + # pull_request run uses the PR's copy of this file, so a PR can edit the workflow to pass. Closing that needs + # a ruleset-required workflow or CODEOWNERS on .github/. + - uses: actions/checkout@v6 + with: + ref: main + sparse-checkout: .github/release + persist-credentials: false + + - name: Classify PR + id: class + env: + TITLE: ${{ github.event.pull_request.title }} + LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} + BASE: ${{ github.base_ref }} + run: | + if [ ! -f .github/release/relmgr.py ]; then # main before the release tooling lands: nothing to check against + echo "::notice title=Lint PR::skipped: main has no .github/release/relmgr.py yet" + exit 0 + fi + python3 -I - <<'PY' # isolated: no working-directory module shadows the stdlib + import json, os, sys + sys.path.insert(0, ".github/release") + import relmgr + + title, labels = os.environ["TITLE"], json.loads(os.environ["LABELS"]) + summary = open(os.environ["GITHUB_STEP_SUMMARY"], "a") + try: + if "release-branch-only" in labels: + if not os.environ["BASE"].startswith("release/"): + raise ValueError("label `release-branch-only` is for PRs into a release/X.Y branch; remove it " + "here (a main change reaches release branches through Backport)") + cls = relmgr.pr_fix_class(title, labels) or "none" + except ValueError as e: + msg = (f"{e}\nFix: title as `[TYPE] text` or `TICKET [TYPE] text` (e.g. `UN-123 [FIX] ...`); allowed labels are " + "fix/security or fix/critical (at most one), and fix/needs-backport.") + print("::error title=Lint PR::" + msg.replace("\n", "%0A")) # workflow commands end at a raw newline + summary.write(f"### Lint PR: failed\n\n{msg}\n") + sys.exit(1) + print(f"class={cls}") + open(os.environ["GITHUB_OUTPUT"], "a").write(f"class={cls}\n") + summary.write(f"### Lint PR\n\nBackport class: `{cls}`\n") + if cls != "none": + summary.write("\nKeep this PR to the fix; a bundled fix cannot be cherry-picked.\n") + PY diff --git a/.github/workflows/state.yaml b/.github/workflows/state.yaml new file mode 100644 index 0000000000..9a97daca3b --- /dev/null +++ b/.github/workflows/state.yaml @@ -0,0 +1,36 @@ +name: Release state + +# Read-only: prints every release/X.Y branch with its state. It writes nothing. +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + state: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # relmgr needs full history and tags + + - name: Derive branch and target state + id: state + env: + GH_TOKEN: ${{ github.token }} + run: | + start=$(date +%s.%N) + python3 .github/release/relstate.py > state.json + echo "relstate took $(echo "$(date +%s.%N) - $start" | bc) s" + cat state.json + + - uses: actions/upload-artifact@v7 + with: + name: relstate + path: state.json + + - name: Summary table + working-directory: .github/release + run: python3 -c 'import json, relstate; print(relstate.summary(json.load(open("../../state.json"))))' >> "$GITHUB_STEP_SUMMARY" From b436f0aea7ec65f61d62deb893f9bea2936261cb Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 10:59:56 +0530 Subject: [PATCH 02/37] feat: add Backport and Release branch CI; latest stable tag only in run-platform.sh Backport picks each fix merged to main onto the release lines and records how people close backport conflict PRs. Release branch CI checks that every commit on a release line is a pick, a decline, release-branch-only or build-generated. Both use the repo's existing GitHub App token for the pushes and PRs that must start other workflows. run-platform.sh now resolves "latest" from plain vX.Y.Z tags only, so the ee-vX.Y.Z(-rc.N) line tags never become a community install. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/backport.yaml | 141 ++++++++++++++++++ .github/workflows/release-branch-ci.yaml | 174 +++++++++++++++++++++++ run-platform.sh | 2 +- 3 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/backport.yaml create mode 100644 .github/workflows/release-branch-ci.yaml diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml new file mode 100644 index 0000000000..ae97a4427b --- /dev/null +++ b/.github/workflows/backport.yaml @@ -0,0 +1,141 @@ +name: Backport +# Expressions have no substring, so a push run names the full sha. +run-name: >- + ${{ github.event_name == 'pull_request_target' + && format('record #{0}', github.event.pull_request.number) + || format('picks {0}', inputs.commits || github.sha) }} + +# Two jobs, one per trigger family. +# pick: every fix merged to main is cherry-picked with -x onto each stabilizing release branch. +# Each push to main rescans every target since its cut (backport.py --rescan) and picks whatever no pick or +# decline trailer names there, so a push run that a newer push replaces while pending costs nothing. +# A dispatch with `commits` picks only those; a rescan never picks a held fix, so a push never replaces a dispatch. Every pick is its own PR with the original PR's title; +# the bot squash-merges clean ones in the same run, conflicts stay open for a person. +# Security/critical fixes reach stabilized branches only with confirm_stabilized. Never builds. +# record: people land a backport conflict PR with the normal buttons (resolve, one approving review, Squash and merge) +# or close it. The bot records what happened (backport_merge.py --pr N): +# merged -> label backport-merged, head deleted; closed unmerged -> a decline: empty decline commit on the base +# (the closer's close comment is the reason), label backport-declined, head kept so the PR can be reopened; +# reopened -> label backport-conflict. Reopen and merge undoes a decline: the newest trailer on the line wins. +# pull_request_target, not pull_request: GitHub runs no pull_request workflow on a PR that conflicts with its base +# at the git level (no merge ref), and a conflict PR whose base later changed the same lines is one. +# pull_request_target runs the base line's copy of this file, so it fires only on lines that carry it. Safe: +# the job checks out main and never runs head code; backport_merge.py refuses a head that is not +# backport/X.Y/ into release/X.Y (an expression cannot match that pattern, so the job filter is coarser). A failed record run is recovered with "Re-run jobs". +# Scripts run from main's tree, like Promote and Take RC build. +on: + push: + branches: [main] + pull_request_target: + types: [closed, reopened] + branches: ["release/**"] + workflow_dispatch: + inputs: + commits: + description: "Fixes to backport (PR numbers or SHAs)" + type: string + required: false + confirm_stabilized: + description: "Include stable branches (security/critical only)" + type: boolean + default: false + branches: + description: "Only these branches (optional)" + type: string + required: false + +jobs: + pick: + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 # backport.py PICK_TIMEOUT: a head with no PR younger than this belongs to a running run + permissions: + contents: read + concurrency: + # Push runs are serialised, and a newer push replaces a pending one: the next rescan picks what it would have. + # A dispatch is its own group, never replaced: it may carry confirm_stabilized, which no rescan has. It can run + # beside a push run or another dispatch. Two runs picking one fix onto one line race on its pick head: the + # loser's push is rejected (retried; skipped if the winner already landed the pick, else a failed row), and a + # head with no open PR yet reads as pending until it is older than timeout-minutes. If the winner merged and + # deleted the head first, the loser can open a second pick PR for a fix already on the line. A squash after + # the base moved is retried by backport_merge.squash. + group: backport-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'rescan' }} + env: + PYTHONDONTWRITEBYTECODE: "1" # the conflict path runs `git add -A` + steps: + - name: Inputs + if: github.event_name == 'workflow_dispatch' + env: + COMMITS: ${{ inputs.commits }} + run: | + if [ "$GITHUB_REF" != refs/heads/main ]; then echo '::error::Run this from main ("Use workflow from: main")'; exit 1; fi + if [ -z "$COMMITS" ]; then echo "::error::Set commits: the PR numbers or SHAs to backport"; exit 1; fi + + # The repo's GitHub App, as in create-release.yaml: a GITHUB_TOKEN push or PR starts no workflow. + - name: Bot token + id: bot + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.PUSH_TO_MAIN_APP_CLIENT_ID }} + private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }} + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # relmgr needs full history and tags + token: ${{ steps.bot.outputs.token }} # picks are pushed as the bot + + - name: Main tip + # A late push run checks out its event sha as origin/main; rescan the real tip instead. + run: git fetch origin +refs/heads/main:refs/remotes/origin/main + + - name: Branch state + env: + GH_TOKEN: ${{ github.token }} + run: | + # Scripts run from outside the tree: the pick checks out release branches, which predate them. + cp .github/release/*.py "$RUNNER_TEMP/" + python3 "$RUNNER_TEMP/relstate.py" > "$RUNNER_TEMP/state.json" + + - name: Pick + env: + GH_TOKEN: ${{ steps.bot.outputs.token }} + COMMITS: ${{ inputs.commits }} + CONFIRM: ${{ inputs.confirm_stabilized }} + BRANCHES: ${{ inputs.branches }} + run: | + if [ "$GITHUB_EVENT_NAME" = push ]; then args=(--rescan); else args=(--commits "$COMMITS"); fi + if [ "$CONFIRM" = true ]; then args+=(--confirm-stabilized); fi + python3 "$RUNNER_TEMP/backport.py" --state "$RUNNER_TEMP/state.json" "${args[@]}" --branches "$BRANCHES" + + record: + if: >- + github.event_name == 'pull_request_target' && startsWith(github.event.pull_request.head.ref, 'backport/') + && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + concurrency: + group: backport-pr-${{ github.event.pull_request.number }} # its own group: in a pick group it could replace a pending pick run + steps: + # The repo's GitHub App, as in create-release.yaml: a GITHUB_TOKEN push or PR starts no workflow. + - name: Bot token + id: bot + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.PUSH_TO_MAIN_APP_CLIENT_ID }} + private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }} + + - uses: actions/checkout@v6 + with: + ref: main + fetch-depth: 0 # the decline reads the pick from the head and the base's history + token: ${{ steps.bot.outputs.token }} + + - name: Record the close + env: + GH_TOKEN: ${{ steps.bot.outputs.token }} # labels, comments, the decline commit and the head delete + PR: ${{ github.event.pull_request.number }} + # who closed it: their close comment is the decline reason + SENDER: ${{ github.event.sender.login }} + run: python3 .github/release/backport_merge.py --pr "$PR" --sender "$SENDER" diff --git a/.github/workflows/release-branch-ci.yaml b/.github/workflows/release-branch-ci.yaml new file mode 100644 index 0000000000..f17eaba860 --- /dev/null +++ b/.github/workflows/release-branch-ci.yaml @@ -0,0 +1,174 @@ +name: Release branch CI + +# Every commit after the cut must be a cherry-pick of a main commit (or its decline), release-branch-only, +# or build-generated. Anything else fails. +# On a PR into release/** this is the required check `check` over the PR's own commits, so a human change +# is checked before it lands. A backport/X.Y/ PR is judged as its squash: only its first commit (the pick, whose +# trailer the squash message keeps) is classified, so a plain resolution commit passes; conflict markers in the +# head's tree fail it. The push run stays as the backstop for bot pushes and alerts on failure. +# A pull_request run uses the workflow file from the PR head: a branch cut before this trigger existed gets no PR +# check, and a PR can edit this file. Job `check` runs there; it holds no secret. +# A PR labelled `release-branch-only` passes with commits of any kind. On the label, job `trailer` pushes one +# empty commit `Release-branch-only: ` onto the PR head (unless a commit already has the line); Squash and +# merge keeps it in the message, and the push run checks that merged commit like any other. It holds the bot token, +# so it runs on pull_request_target: the base line's copy of this file, which a PR cannot edit. A line cut before +# that trigger existed gets no bot commit; add the line to a commit yourself there. +# Label changes on a closed PR run nothing. +# No push builds an RC: a security or critical pick lands and waits for the RM's Take RC build, like any other pick. +on: + push: + branches: ["release/**"] + pull_request: + branches: ["release/**"] + types: [opened, synchronize, reopened, labeled, unlabeled] # the label changes the verdict + pull_request_target: + branches: ["release/**"] + types: [labeled] # job trailer only + +permissions: + contents: read + +jobs: + check: + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.state != 'closed') + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + BRANCH: ${{ github.base_ref || github.ref_name }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 # relmgr needs full history and tags + persist-credentials: false # the head is only read + + # The classifier comes from main, so a PR's edit to relmgr cannot pass its own check. That guards relmgr + # only: a PR can still edit this workflow file, which its pull_request run uses, to pass `check`. Closing + # that needs a ruleset-required workflow or CODEOWNERS on .github/. + - uses: actions/checkout@v6 + with: + ref: main + path: .relmgr + sparse-checkout: .github/release + persist-credentials: false + + - name: Check commit kinds + id: classify + env: + EVENT: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + RBO_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'release-branch-only') }} + HEAD_REF: ${{ github.head_ref }} + run: | + python3 -I - <<'PY' # isolated: a module file in the head checkout never shadows the stdlib + import os, sys + sys.path.insert(0, ".relmgr/.github/release") + import relmgr as rm + repo, branch = os.environ["GITHUB_WORKSPACE"], f"origin/{os.environ['BRANCH']}" + try: + rows, where = rm.ci_commits(repo, branch, os.environ["EVENT"], os.environ["BEFORE"], os.environ["HEAD_REF"], + os.environ["RBO_LABEL"] == "true") + except ValueError as e: + sys.exit(f"::error::{e}") + for c, kind in rows: + print(f"{'::error::' if kind == 'invalid' else ''}{c[:12]} {kind:20} {rm.git(repo, 'log', '-1', '--format=%s', c)}") + bad = sum(kind == "invalid" for _, kind in rows) + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"invalid={bad}\npicks={' '.join(c for c, kind in rows if kind == 'cherry-pick')}\n") + sys.exit(f"{bad} invalid commit(s) on {where}" if bad else 0) + PY + + - name: Merge note + if: always() && github.event_name == 'pull_request' + env: + RBO_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'release-branch-only') }} + INVALID: ${{ steps.classify.outputs.invalid || '0' }} + run: | + if [ "$INVALID" != 0 ]; then + hint="A commit here is not a pick from main. A change only for $BRANCH: add the \`release-branch-only\` label. A fix on main: cherry-pick it with \`git cherry-pick -x\`." + echo "::notice::$hint" + echo "$hint" >> "$GITHUB_STEP_SUMMARY" + elif [ "$RBO_LABEL" = true ]; then + echo "Squash and merge: the \`Release-branch-only: \` commit the bot added carries the line." >> "$GITHUB_STEP_SUMMARY" + else + echo "Merge with squash; the squash message keeps each commit's trailer (a backport PR's from its first commit)." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Branch state (informational) + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: python3 .relmgr/.github/release/relstate.py | jq --arg b "$BRANCH" '.branches[] | select(.branch == $b)' + + # On the `release-branch-only` label: one empty bot commit carrying `Release-branch-only: ` (ticket from + # the title, else `#`), so the squash message has the line. A commit already carrying it -> nothing pushed. The push fires + # `synchronize`, so `check` runs once more. A fork head cannot take a bot push: fail and say so. + # pull_request_target: this job is the base line's copy. The head is only data here: read without a token, its + # files never imported, and the push ignores its git config. + trailer: + if: github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name == 'release-branch-only' + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + BASE_REF: ${{ github.base_ref }} + TITLE: ${{ github.event.pull_request.title }} + PR: ${{ github.event.pull_request.number }} + steps: + - name: Same-repo head + run: | + [ "$HEAD_REPO" = "$GITHUB_REPOSITORY" ] || { echo "::error::release-branch-only: the head is on $HEAD_REPO, a fork; the bot cannot push the \`Release-branch-only:\` commit there. Push the branch to $GITHUB_REPOSITORY and open the PR from it, or add the line to a commit yourself."; exit 1; } + + # The head is checked out without the bot token; only the push below gets it. The logic comes from main. + # Run on the head's tip SHA, so a push to the head after the label makes the bot push fail, not skip a check. + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 # every branch, so origin/ is current + persist-credentials: false + + - uses: actions/checkout@v6 + with: + ref: main + path: .relmgr + sparse-checkout: .github/release + persist-credentials: false + + - name: Add the Release-branch-only commit + id: rbo + run: | + python3 -I - <<'PY' # isolated: a module file in the head checkout never shadows the stdlib + import os, subprocess, sys + sys.path.insert(0, ".relmgr/.github/release") + import relmgr as rm + ticket = rm.rbo_key(os.environ["TITLE"], os.environ["PR"]) + msgs = rm.git(".", "log", "--format=%B", f"origin/{os.environ['BASE_REF']}..HEAD") + if have := rm.RBO_RE.findall(msgs): + print(f"A commit on the PR already has Release-branch-only: {', '.join(sorted(set(have)))}; nothing pushed.") + sys.exit(0) + subprocess.run(["git", "-c", "core.hooksPath=/dev/null", "-c", "user.name=releases-bot", "-c", f"user.email={rm.BOT_EMAIL}", + "commit", "--allow-empty", "-m", f"Release-branch-only: {ticket}"], check=True) + print(f"Committed {rm.git('.', 'rev-parse', 'HEAD')[:12]} Release-branch-only: {ticket}") + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write("push=true\n") + PY + + # The repo's GitHub App, as in create-release.yaml: a GITHUB_TOKEN push or PR starts no workflow. + - name: Bot token + id: bot + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.PUSH_TO_MAIN_APP_CLIENT_ID }} + private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }} + + - name: Push it + if: steps.rbo.outputs.push == 'true' + env: + BOT_TOKEN: ${{ steps.bot.outputs.token }} # a GITHUB_TOKEN push would not start the `check` re-run + run: | + # No hooks, and an explicit URL rather than the checkout's remote config. + helper='!f() { echo username=x-access-token; echo "password=$BOT_TOKEN"; }; f' + git -c core.hooksPath=/dev/null -c credential.helper= -c credential.helper="$helper" \ + push "https://github.com/${GITHUB_REPOSITORY}.git" "HEAD:refs/heads/$HEAD_REF" diff --git a/run-platform.sh b/run-platform.sh index 7ce14347bc..a7431fc4c8 100755 --- a/run-platform.sh +++ b/run-platform.sh @@ -164,7 +164,7 @@ do_git_pull() { git fetch --quiet --tags if [[ "$opt_version" == "latest" ]]; then - target_branch=`git ls-remote --tags origin | awk -F/ '{print $3}' | sort -V | tail -n1` + target_branch=`git ls-remote --tags origin | awk -F/ '{print $3}' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n1` elif [[ "$opt_version" == "main" ]]; then target_branch="main" opt_build_local=true From 96fd33a460501fba5a9ecffd643b2c26d865aef9 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 12:19:38 +0530 Subject: [PATCH 03/37] fix: sync .github/release with the enterprise repo's copy Registry reads move to Artifact Registry behind RELSTATE_AR_* variables that the calling workflow sets, so no registry location lives in this directory. OSS lines still read their own git tags. Includes that copy's review fixes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 35 ++-- .github/release/backport_merge.py | 11 +- .github/release/convgate.py | 6 +- .github/release/oss.py | 72 +++++-- .github/release/relmgr.py | 38 +++- .github/release/relstate.py | 157 ++++++++++----- .github/release/test_backport.py | 103 +++++++++- .github/release/test_backport_merge.py | 28 ++- .github/release/test_convgate.py | 34 ++-- .github/release/test_links.py | 6 +- .github/release/test_oss.py | 99 +++++++++ .github/release/test_relstate.py | 269 ++++++++++++++++++++----- 12 files changed, 688 insertions(+), 170 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index be27dc998e..949a7be6ae 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -6,9 +6,11 @@ --rescan (every push to main): ignore the pushed commits and pick, for each target, every main commit since its cut that no pick or decline trailer there names: plain fixes too on fix targets (held accept=OWED), security/critical only on stabilized ones (accept=HELD, held for the RM without --confirm-stabilized). A run cancelled by a newer push -costs nothing: the next run rescans. A decline carries the pick trailer, so a declined fix is never re-picked; an -open conflict PR is left alone (`skipped: conflict PR pending`) until a person merges or closes it; an open clean -pick PR whose merge failed in an earlier run is merged by the next run. +costs nothing: the next run rescans. The exception is a run stopped after pushing a pick head and before opening +its PR: after PICK_TIMEOUT that head fails later runs until a person deletes it (finish_open). +A decline carries the pick trailer, so a declined fix is never re-picked; an open conflict PR is left alone +(`skipped: conflict PR pending`) until a person merges or closes it; an open clean pick PR whose merge failed in +an earlier run is merged by the next run. --commits (dispatch): only these commits. Run from a full clone whose `origin` pushes with the bot token, with GH_TOKEN set to the bot token. @@ -257,7 +259,8 @@ def this_run(rows): out = [] for sha, pr, _, b, res, _ in rows: src = link_pr(pr) if pr.startswith("#") else rm.commit_link(sha) - if m := re.fullmatch(r"merged (\w+) \(#(\d+)\)", res): + # finish_open adds ", retried" when it merges a pick PR an earlier run left open. + if m := re.fullmatch(r"merged (\w+) \(#(\d+)(?:, retried)?\)", res): out.append( f"- {src} → `{b}`: merged as {rm.commit_link(m[1])} ({rm.pr_link(int(m[2]))})" ) @@ -596,14 +599,8 @@ def rescan(slug, targets, confirm, listed): } print(f"rescan: {listed_n} PRs listed, {len(classes) - listed_n} fallback lookup(s)") plan, awaiting, unclassifiable = rescan_plan(targets, held_by_branch, confirm) - for ( - sha, - b, - pr, - title, - ) in ( - unclassifiable - ): # a human dispatches with a class or declines it; re-warned each push + # A human fixes the merged PR's title or adds a fix/ label (declines it when there is no PR); re-warned each push. + for sha, b, pr, title in unclassifiable: print( f"::warning::{sha[:7]} ({pr}, {title}) cannot be classed for {b}: not picked" ) @@ -686,6 +683,7 @@ def push_head(slug, sha, branch, head, src, sleep=time.sleep): f"Rerun Backport for {sha[:7]} on {branch}." ) print(f"::error::{sentence}") + # Not de-duplicated: a push rejected on every run comments on every run (convgate reads only the newest). if url and re.fullmatch(r"#\d+", src): soft( f"comment on {src}", @@ -799,6 +797,8 @@ def finish_open(slug, branch, head): def pick(slug, sha, branch, title, src): """Returns (outcome, detail); outcome in merged | pr | skipped | failed. title: PR title; src: original PR ref.""" remote = f"origin/{branch}" + # Fresh tip: an overlapping run may have merged this pick after this run fetched. + run("git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}") if rm.is_converged(REPO, sha, remote, main="origin/main"): return "skipped", "already on branch" head = f"backport/{branch.split('/', 1)[1]}/{sha[:7]}" @@ -852,7 +852,13 @@ def pick(slug, sha, branch, title, src): "git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}" ) # later picks stack on it return "merged", f"{rm.git(REPO, 'rev-parse', '--short=7', remote)} (#{n})" - files = rm.git(REPO, "diff", "--name-only", "--diff-filter=U").split() + # -z: a path with a space or a non-ASCII name comes back unquoted and in one piece (rm.git's strip can still + # drop a leading space on the first path; only the commit message and PR body read this list). + files = [ + f + for f in rm.git(REPO, "diff", "--name-only", "-z", "--diff-filter=U").split("\0") + if f + ] if not files: # not a conflict a human can resolve on a PR empty = rm.git( REPO, "rev-parse", "-q", "--verify", "CHERRY_PICK_HEAD", check=False @@ -874,7 +880,8 @@ def pick(slug, sha, branch, title, src): with open(os.path.join(REPO, ".git", "MERGE_MSG")) as f: msg = "".join(line for line in f if not line.startswith("#")).strip() msg += "\n\nConflicts:\n" + "".join(f"\t{p}\n" for p in files) - run("git", "add", "-A") + # -u: tracked and unmerged paths only; -A would also commit untracked files such as state.json. + run("git", "add", "-u") run("git", "commit", "-q", "--no-verify", "-F", "-", input=msg) if failed := push_head(slug, sha, branch, head, src): return failed diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 628bfaaa6c..b4545893b3 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -61,6 +61,10 @@ def decline_reason(comments, sender, closed_at): for c in sorted(comments, key=lambda c: c["created_at"], reverse=True): if c["user"] == sender and end - REASON_WINDOW <= at(c["created_at"]) <= end: line = (c["body"].strip().splitlines() or [""])[0] + # A pick trailer in the reason would mark that other commit declined in the decline commit's message. + # Strip until none is left: a trailer nested in another reassembles after one pass. + while rm.PICK_RE.search(line): + line = rm.PICK_RE.sub("", line) return line.removeprefix("Declined:").strip() or None return None @@ -149,9 +153,10 @@ def squash(slug, pr, msg, sleep=time.sleep): ) return True except RuntimeError as e: - if ( - "status check" in str(e).lower() - ): # a required check GitHub still waits for: the caller reports it + # A required check GitHub still waits for: the caller reports it. Accepted: this matches GitHub's + # "Required status check ... is expected/in progress" wording, which GitHub does not document as stable. + if "status check" in str(e).lower(): + print(f"#{pr['number']}: {str(e).splitlines()[0]}") return False if wait is None or "base branch was modified" not in str(e).lower(): raise diff --git a/.github/release/convgate.py b/.github/release/convgate.py index 876a5472a9..3542234985 100644 --- a/.github/release/convgate.py +++ b/.github/release/convgate.py @@ -4,7 +4,7 @@ Usage: convgate.py --branch release/X.Y --target T [--promote-ref vX.Y.Z-rc.N] [--state state.json] [--repo DIR] [--oss-repo DIR] [--annotate warning|error] [--out FILE] -Wraps relmgr.convergence_report unchanged and adds four things: +Wraps relmgr.convergence_report unchanged and adds: - scope: relstate `supported[T]` minus the branch. Retired and not-enabled branches are out by construction. - direction: an older branch's picks are checked against the newer branch, whichever side is being promoted. - reference: a newer branch with a promoted stable tag for T is scanned at that tag (what ships next), not at its tip, @@ -162,6 +162,8 @@ def oss_pins(repo, oss_repo, branch, target, scope, ref, rows, annotate): main=main, newer_branch=oss_refs.get(line_new), ) + # Accepted: only CONVERGED and MISSING count for a pin; DECLINED, UNCLASSIFIABLE and RBO-ABSENT + # OSS picks are left out of the counts and the detail. if r[1] in ("CONVERGED", "MISSING") ] ) @@ -416,6 +418,8 @@ def cell(parts, none="-"): return "
".join(parts).replace("|", "\\|") or none def fixes(name, ls): + # Accepted: a line whose only finding is UNCLASSIFIABLE has no todo, so it is counted as converged here and + # the row reads ✅. The line's summary in the folded gate detail (out) still counts them. named = [x for x in ls if x["state"] == "stabilizing" or x["todo"] or x["rbo"]] rest = [x for x in ls if x not in named] counts = [f"`{x['line']}` {x['summary']}" for x in named] diff --git a/.github/release/oss.py b/.github/release/oss.py index 349ea30663..9e051143c9 100644 --- a/.github/release/oss.py +++ b/.github/release/oss.py @@ -23,7 +23,7 @@ import relstate as rs OSS = rs.PINS or "Zipstack/unstract" # the OSS repo the cloud lines pin -PREFIX = rs.REPOS[OSS.split("/")[1].lower()][5] +PREFIX = rs.REPOS[OSS.split("/")[1].lower()][4] VER = rs.ver_re(PREFIX) PIN_RE = re.compile(r'^unstract-platform-version = "([^"]*)"$', re.M) HERE = os.path.dirname(os.path.abspath(__file__)) @@ -104,9 +104,24 @@ def _push(oss, *refspecs): rm.git(oss, "push", "-q", "--atomic", "origin", *refspecs) -def _bump(oss, branch, base, tag, slack_url): +def _tag_push(oss, name, sha, msgs, *refspecs): + """Tag sha as name (_tag) and push refspecs. A failed push drops the tag when this call made it, so a re-run in + this clone does not take a tag origin never got as already done. + """ + new = _tag(oss, name, sha, *msgs) + try: + _push(oss, *refspecs) + except RuntimeError: + if new: + rm.git(oss, "tag", "-d", name) + raise + + +def _bump(oss, branch, base, tag, slack_url, old): """One empty bot commit `release: ` on base, tagged, pushed as branch's new tip with its tag. The commit gives - the RC its own commit, so a line never shares a tag's commit with main or another line. + the RC its own commit, so a line never shares a tag's commit with main or another line. The push is leased on + old, the branch tip it read: for rc, a branch deleted or moved since then fails it instead of being re-created or + overwritten; for cut, old is "" (no branch), so a branch created since then fails it. """ sha = _bot( oss, @@ -117,8 +132,15 @@ def _bump(oss, branch, base, tag, slack_url): "-m", f"release: {tag.removeprefix('v')}", ) - _tag(oss, tag, sha, *([f"Slack: {slack_url}"] if slack_url else [])) - _push(oss, f"{sha}:refs/heads/{branch}", f"refs/tags/{tag}") + _tag_push( + oss, + tag, + sha, + [f"Slack: {slack_url}"] if slack_url else [], + f"--force-with-lease=refs/heads/{branch}:{old}", + f"{sha}:refs/heads/{branch}", + f"refs/tags/{tag}", + ) rm.git(oss, "update-ref", f"refs/remotes/origin/{branch}", sha) return sha @@ -191,7 +213,12 @@ def cut(oss, line, slack_url=""): f"OSS minor {line} already has tags ({', '.join(sorted(used, key=rs._vkey))})" ) _bump( - oss, branch, rm.git(oss, "rev-parse", "refs/remotes/origin/main"), tag, slack_url + oss, + branch, + rm.git(oss, "rev-parse", "refs/remotes/origin/main"), + tag, + slack_url, + "", ) return tag @@ -205,7 +232,7 @@ def rc(oss, branch, slack_url=""): raise RuntimeError(f"OSS {branch} does not exist: cloud Cut makes it") if not (tag := choose(tags_on(oss, sha, branch))): tag = rm.rc_version(oss, f"refs/remotes/origin/{branch}", PREFIX) - _bump(oss, branch, sha, tag, slack_url) + _bump(oss, branch, sha, tag, slack_url, sha) print(f"OSS {branch}: new RC {tag}", file=sys.stderr) if "-rc." in tag and gate(oss, branch, tag, "warning") == 1: print( @@ -269,14 +296,19 @@ def promote(repo, oss, cloud_rc, force=False, reason=""): + unchecked + ([f"open conflict PRs: {', '.join(prs)}"] if prs else []) ) - _tag(oss, stable, sha, f"promote: {pinned} -> {stable} (cloud {cloud_rc})", *forced) - _push(oss, f"refs/tags/{stable}") + _tag_push( + oss, + stable, + sha, + [f"promote: {pinned} -> {stable} (cloud {cloud_rc})", *forced], + f"refs/tags/{stable}", + ) return stable def retire(oss, branch, reason, dry_run=True): - """Retire OSS release/X.Y with its cloud line: archive tag retired/X.Y-, then (no OSS GA) its RC tags, - then the branch. With a GA its tags stay and the minor is never reused; without one it is. Summary lines. + """Retire OSS release/X.Y with its cloud line, in one atomic push: archive tag retired/X.Y-, delete + (no OSS GA) its RC tags and the branch. With a GA its tags stay and the minor is never reused; without one it is. Summary lines. """ line = branch.removeprefix("release/") sha = tip(oss, branch) @@ -313,11 +345,19 @@ def retire(oss, branch, reason, dry_run=True): ] if dry_run: return out - if _tag(oss, archive, sha, f"retired {branch}: {reason}"): - _push(oss, f"refs/tags/{archive}") - if rcs: - _push(oss, *(f":refs/tags/{t}" for t in rcs)) - _push(oss, f":refs/heads/{branch}") + # One atomic push leased on the tip read above: a commit pushed since then fails the whole retire, so on origin + # the branch, its RC tags and the archive tag all stay as they were. The archive tag is always sent (a no-op when + # origin has it), so a re-run in this clone archives the tip too. + _tag_push( + oss, + archive, + sha, + [f"retired {branch}: {reason}"], + f"--force-with-lease=refs/heads/{branch}:{sha}", + f"refs/tags/{archive}", + *(f":refs/tags/{t}" for t in rcs), + f":refs/heads/{branch}", + ) return out diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py index 6b00a49c73..c178a6a0d2 100644 --- a/.github/release/relmgr.py +++ b/.github/release/relmgr.py @@ -59,14 +59,16 @@ def run_link(run_id): def image_link(target, tag): - """The GHCR package page of //release, the tag in the text.""" - text = f"`{target}/release:{tag}`" - if not _web(): - return text - owner, repo = os.environ["GITHUB_REPOSITORY"].split("/") + """The Artifact Registry console page of target's repository (RELSTATE_AR_), the tag in the text; + plain text when that variable is unset. + """ + text = f"`{target}:{tag}`" + repo = os.environ.get("RELSTATE_AR_" + target.upper().replace("-", "_"), "") + m = re.fullmatch(r"projects/([^/]+)/locations/([^/]+)/repositories/([^/]+)", repo) return ( - f"[{text}]({os.environ['GITHUB_SERVER_URL']}/orgs/{owner}/packages/container/package/" - f"{repo.lower()}%2F{target}%2Frelease)" + f"[{text}](https://console.cloud.google.com/artifacts/docker/{m[1]}/{m[2]}/{m[3]})" + if m + else text ) @@ -181,7 +183,8 @@ def classify_commit(repo, sha, bot_email=BOT_EMAIL, main="main"): def pr_commits(repo, base, head_ref, head="HEAD"): """The commits the PR check classifies on a PR from head_ref into base. A backport/* PR lands as its squash, whose message keeps the first commit's pick trailer: only that commit is judged, so a plain resolution commit after it - passes. Any other PR: every commit in base..head. + passes. This relies on release/* allowing squash merges only (a ruleset, set before Backport runs); a rebase + merge would land the unjudged commits as they are. Any other PR: every commit in base..head. """ out = git(repo, "rev-list", "--reverse", f"{base}..{head}") commits = out.split() if out else [] @@ -284,9 +287,10 @@ def pr_fix_class(title, labels=()): if len(classes) > 1: raise ValueError(f"conflicting fix labels: {sorted('fix/' + c for c in classes)}") # A revert takes the reverted PR's class: it must follow the original onto every branch that carries it. - inner = title + # Stripped: GitHub keeps a title's leading space, and a typo'd space must not leave a merged fix unclassifiable. + inner = title.strip() while r := REVERT_RE.match(inner): - inner = r.group(1) + inner = r.group(1).strip() m = PR_TITLE_RE.match(inner) if not m and not set(labels) & set(FIX_LABELS): raise ValueError( @@ -302,11 +306,23 @@ def pr_fix_class(title, labels=()): ) +def release_only_refusal(base, labels): + """Why Lint PR refuses the release-branch-only label on a PR into `base`, or None. The label is for PRs into + a release/X.Y branch; a main change reaches release branches through Backport. + """ + if "release-branch-only" in labels and not base.startswith("release/"): + return ( + "label `release-branch-only` is for PRs into a release/X.Y branch; remove it here " + "(a main change reaches release branches through Backport)" + ) + return None + + def rbo_key(title, pr): """The `Release-branch-only:` value for PR `pr`: the title's ticket, else `#`. A `#` key matches no other line's, so the convergence report always lists that change on the newer line for a human to check. """ - m = PR_TITLE_RE.match(title) + m = PR_TITLE_RE.match(title.strip()) return m["ticket"] if m and m["ticket"] else f"#{pr}" diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 316460516e..d2c9e63b1a 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -9,47 +9,49 @@ import argparse import datetime as dt +import http.client import json import os import re -import subprocess import sys import time +import traceback +import urllib.error import urllib.parse +import urllib.request import relmgr as rm -WINDOW = dt.timedelta( - days=184 -) # on-prem support window from the chart push date: 6 months + 1 day buffer +# On-prem support window from the line's latest stable chart push, so each patch restarts it. +WINDOW = dt.timedelta(days=184) # 184 days covers the longest 6-month span WINDOW_LINES = ( 6 # and only the newest 6 on-prem GA lines are supported, whichever ends first ) -# Per repo, so both repos run this same file: (probe artifact, targets, targets where only the latest GA line is -# supported, targets supported while the branch exists: retired only when another repo's Retire names the line, -# repo whose tags this repo's lines pin in pyproject.toml or None, tag prefix of this repo's lines). -# PROBE is an artifact known to exist; it proves the token can read packages (a 404 alone is ambiguous). +# Per repo, so both repos run this same file: (targets, targets where only the latest GA line is supported, +# targets supported while the branch exists: retired only when another repo's Retire names the line, repo whose +# tags this repo's lines pin in pyproject.toml or None, tag prefix of this repo's lines). +# Artifacts are / in Artifact Registry; RELSTATE_AR_ names the repository (ar_repository). # TARGETS: target -> (artifact holding RC / promoted builds, GA anchor artifact). Cloud GA = the vX.Y.Z retag # of the build artifact; on-prem GA = the chart push. GIT instead of an artifact: the line's own git tags, the stable # tag's date as the GA date (OSS lines: cloud builds them from source, so they have no artifact). # OSS lines are tagged ee-vX.Y.Z(-rc.N); plain vX.Y.Z tags there are community releases from main, never a line's. GIT = "git" REPOS = { + # Cloud: each target's build artifact is backend, since every build pushes it (all services share one tag). "unstract-cloud": ( - "cloud/smoke", { - "cloud": ("cloud/release", "cloud/release"), - "on-prem": ("on-prem/release", "on-prem/chart"), + "cloud": ("cloud/backend", "cloud/backend"), + "on-prem": ("on-prem/backend", "chart/unstract-platform"), }, {"cloud"}, set(), "Zipstack/unstract", "", ), - "unstract": (None, {"oss": (GIT, GIT)}, set(), {"oss"}, None, "ee-"), + "unstract": ({"oss": (GIT, GIT)}, set(), {"oss"}, None, "ee-"), } -OWNER, PKG_PREFIX = ( - os.environ.get("GITHUB_REPOSITORY", "zipstack/unstract-cloud").lower().split("/") +SLUG = ( + os.environ.get("GITHUB_REPOSITORY", "zipstack/unstract-cloud").lower().split("/")[1] ) @@ -59,52 +61,102 @@ def ver_re(prefix): def use(slug): """Load the REPOS config of slug (a repo name) into the module globals.""" - global PROBE, TARGETS, LATEST_ONLY, KEPT, PINS, PREFIX, VER_RE + global TARGETS, LATEST_ONLY, KEPT, PINS, PREFIX, VER_RE if slug not in REPOS: raise RuntimeError( f"relstate has no config for repo {slug!r} (GITHUB_REPOSITORY); known: {', '.join(REPOS)}" ) - PROBE, TARGETS, LATEST_ONLY, KEPT, PINS, PREFIX = REPOS[slug] + TARGETS, LATEST_ONLY, KEPT, PINS, PREFIX = REPOS[slug] VER_RE = ver_re(PREFIX) -use(PKG_PREFIX) +use(SLUG) -def _gh(*args): - return subprocess.run( - ["gh", "api", "--paginate", *args], capture_output=True, text=True - ) +AR_API = "https://artifactregistry.googleapis.com/v1" + + +def ar_repository(kind): + """The Artifact Registry repository resource (projects/

/locations//repositories/) of an artifact kind, + from RELSTATE_AR_: the calling workflow sets it, so no registry location lives in this file. + """ + var = "RELSTATE_AR_" + kind.upper().replace("-", "_") + if not (repo := os.environ.get(var)): + raise RuntimeError( + f"registry read {kind}/*: set {var} to projects/

/locations//repositories/" + ) + return repo -def _versions_url(artifact): - return f"/orgs/{OWNER}/packages/container/{urllib.parse.quote(f'{PKG_PREFIX}/{artifact}', safe='')}/versions?per_page=100" +def _ar_get(url, what): + """GET an Artifact Registry API url as JSON, or None when it answers 404. Raises on any other failure.""" + if not (token := os.environ.get("RELSTATE_AR_TOKEN")): + raise RuntimeError( + f"registry read {what}: set RELSTATE_AR_TOKEN to an access token" + ) + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return json.load(r) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + try: + detail = e.read().decode(errors="replace")[:300] + except (OSError, http.client.HTTPException) as r: # the error body itself cut off + detail = repr(r) + raise RuntimeError(f"registry read {what}: HTTP {e.code}: {detail}") from e + except ( + OSError, + http.client.HTTPException, + ValueError, + ) as e: # network (URLError), timeout, a cut-off response, bad JSON + raise RuntimeError(f"registry read {what}: {e!r}") from e + + +def _ar_versions(artifact): + """Every version of artifact (/) with its tags, or None when its first page answers 404.""" + kind, package = artifact.split("/", 1) + url = f"{AR_API}/{ar_repository(kind)}/packages/{urllib.parse.quote(package, safe='')}/versions" + out, page = [], "" + while True: + q = {"view": "FULL", "pageSize": 1000, **({"pageToken": page} if page else {})} + if (body := _ar_get(f"{url}?{urllib.parse.urlencode(q)}", artifact)) is None: + if page: # a later page vanished: the list is incomplete, not empty + raise RuntimeError(f"registry read {artifact}: HTTP 404 on page {page}") + return None + out += body.get("versions", []) + if not (page := body.get("nextPageToken")): + return out def registry_tags(artifact): - """{tag: {digest, pushed_at}} for one GHCR artifact of this repo. Raises on any read failure. + """{tag: {id, digest, pushed_at}} for one Artifact Registry artifact (/). Raises on any read + failure. pushed_at is the version's createTime, to the second. - A 404 is only read as "artifact not pushed yet" when the token can read PROBE; otherwise it may be - a token that cannot read packages, and an empty registry would read as "nothing enabled". + A 404 is only read as "artifact not pushed yet" when the artifact's own repository reads; otherwise it may be + a wrong RELSTATE_AR_, and an empty registry would read as "nothing enabled". """ - r = _gh( - _versions_url(artifact), - "--jq", - ".[] | {id, digest: .name, pushed_at: .created_at, tags: .metadata.container.tags}", - ) - if r.returncode: - if "HTTP 404" not in r.stderr: - raise RuntimeError(f"registry read {artifact}: {r.stderr.strip()}") - if artifact == PROBE or _gh(_versions_url(PROBE), "--jq", "length").returncode: + versions = _ar_versions(artifact) + if versions is None: + repo = ar_repository(artifact.split("/", 1)[0]) + if _ar_get(f"{AR_API}/{repo}", artifact) is None: raise RuntimeError( - f"registry read {artifact}: 404 and probe {PROBE} unreadable: {r.stderr.strip()}" + f"registry read {artifact}: 404 and repository {repo} not found" ) return {} out = {} - for line in r.stdout.splitlines(): - v = json.loads(line) - for tag in v["tags"] or []: - out[tag] = {"id": v["id"], "digest": v["digest"], "pushed_at": v["pushed_at"]} + for v in versions: + pushed = re.sub( + r"\.\d+(?=Z$)", "", v["createTime"] + ) # AR returns up to 9 fraction digits + for t in v.get("relatedTags") or []: + tag = t["name"].rsplit("/tags/", 1)[1] + out[tag] = { + "id": v["name"], + "digest": v["name"].rsplit("/versions/", 1)[1], + "pushed_at": pushed, + } return out @@ -207,8 +259,8 @@ def rc_slack(repo, tag): def target_state(t, name, rows, now): """State of target t on an enabled branch. The one place the GA/supported convention lives. - On-prem: stabilized for WINDOW after the chart push, and only while among the newest WINDOW_LINES lines with an - on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch + On-prem: stabilized for WINDOW after the line's latest stable chart push (a patch restarts it), and only while + among the newest WINDOW_LINES lines with an on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilized from its GA while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. """ @@ -244,9 +296,12 @@ def _main(repo): def _next_minor(repo, ordered): """One minor past the highest release/X.Y line, shipped retired line or stable vX.Y.Z on main, so a cut never - reuses a shipped minor. A line retired early before any GA (retired/X.Y-, its RCs deleted) is reused. + reuses a minor shipped by this flow. A line retired early before any GA (retired/X.Y-, its RCs deleted) is + reused. - With no release/X.Y branch yet (cutover) only the tags count. + With no release/X.Y branch yet (cutover) only the tags count. A stable tag that is not on main does not count + (that keeps strays such as v100.0.x out), so an old-flow release not yet merged back can be proposed again: + check the proposal against `git tag -l` before the first cut. """ main = _main(repo) lines = [ @@ -305,10 +360,11 @@ def derive(repo, now, registry=registry_tags, rc=None): if not m: continue # not a release version (e.g. smoke tags) name = f"release/{m[1]}.{m[2]}" - if name not in refs and shipped.setdefault( - name, bool(rm._tags(repo, m[1], m[2], PREFIX)[0]) - ): - continue + if name not in refs: + if name not in shipped: # one git tag read per line, not per registry tag + shipped[name] = bool(rm._tags(repo, m[1], m[2], PREFIX)[0]) + if shipped[name]: + continue sha = _tag_commit(repo, tag) if name not in refs or not sha or not rm.is_ancestor(repo, sha, refs[name]): why = ( @@ -338,6 +394,8 @@ def derive(repo, now, registry=registry_tags, rc=None): if not tt: continue tt["rcs"].sort(key=_vkey) + # On-prem GA reads the chart artifact: RC chart pushes are tagged 0.0.0-rc.N and match no line; the + # stable chart push is tagged vX.Y.Z (push-charts-to-gcr.yaml checks it against the release tag). stable = re.compile(rf"{re.escape(PREFIX)}v{re.escape(row['line'])}\.\d+") # ga anchors state and the support window; promoted (the build artifact's latest stable) is what ships next for key, artifact in (("ga", anchor), ("promoted", builds)): @@ -418,7 +476,7 @@ def abandon_plan(repo, branch, registry=registry_tags): {"ga": [...], "used": [...], "rc_tags": [...], "versions": {artifact: [{id, digest, tags}]}, "kept": [...]}. A non-empty ga (git stable tag or GA-anchor registry tag) or used (commits that are not bot, pick or decline) refuses the abandon. A registry version is deleted only when every tag on it is an RC of - this line. GitHub refuses to delete a package's last version: such a version is kept and listed. + this line. An artifact's last tagged version is kept and listed, so an abandon never empties an artifact. """ x, y = rm._line(branch) rc_re = re.compile(rf"{re.escape(PREFIX)}v{x}\.{y}\.\d+-rc\.\d+") @@ -565,6 +623,7 @@ def main(argv=None, registry=registry_tags): except ( Exception ) as e: # registry or git failure: no JSON, never a silently empty state + traceback.print_exc() print(f"relstate: {e}", file=sys.stderr) return 1 print(json.dumps(state, indent=2)) diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index dfd93f8c4c..1f5f93eb74 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -1,4 +1,4 @@ -"""backport helpers: held_filter and the held summary line; PR body of a pick PR; picks in main order; branches filter; accept= and UNCLASSIFIABLE; owed-picks summary; dispatch route for a non-fix; push summary (status rows, latest tag, stale mark, quiet line); push rescan plan; PR listing classes; push retry; stuck clean pick PR; open conflict PRs per line. Run: python test_backport.py""" +"""backport helpers: held_filter and the held summary line; PR body of a pick PR; picks in main order; branches filter; accept= and UNCLASSIFIABLE; owed-picks summary; dispatch route for a non-fix; push summary (status rows, latest tag, stale mark, quiet line); push rescan plan; PR listing classes; push retry; stuck clean pick PR; open conflict PRs per line; pick against real git (no untracked file in a conflict commit, a path with a space, a pick merged after the fetch). Run: python test_backport.py""" import io import json @@ -483,12 +483,14 @@ def test_this_run_and_pending(self): "-", ), (a, "#251", "fix", "release/0.189", "skipped: already on branch", "-"), + (a, "#251", "fix", "release/0.188", "merged 89abcde (#262, retried)", "-"), ] self.assertEqual( bp.this_run(rows), [ "- #251 → `release/0.192`: merged as `1234567` (#260)", "- #251 → `release/0.190`: conflict, #261 to resolve", + "- #251 → `release/0.188`: merged as `89abcde` (#262)", ], ) out = bp.pending_block( @@ -675,6 +677,9 @@ def test_clean_pick_merged_with_its_trailer(self): self.assertEqual(out[0], "merged") self.assertEqual(self.merged[0][0], 4) self.assertIn("(cherry picked from commit " + "a" * 40 + ")", self.merged[0][1]) + # The push summary parses this detail: a format change must not drop the row from it. + row = ("a" * 40, "#1", "fix", "release/1.0", " ".join(out), "-") + self.assertEqual(len(bp.this_run([row])), 1) def test_waits_for_a_person(self): pending = ("skipped", "conflict PR pending on `backport/1.0/aaaaaaa`") @@ -923,5 +928,101 @@ def commit(msg, **files): self.assertEqual(rel(c_d="one\n"), "relevant") # a path with a space +class PickGit(unittest.TestCase): + """pick() against real git: a conflict commit carries no untracked file, and a pick another run merged after + this run fetched is skipped. + """ + + PATH = "f" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + ident = { + "GIT_AUTHOR_NAME": "Dev", + "GIT_AUTHOR_EMAIL": "dev@example.com", + "GIT_COMMITTER_NAME": "Dev", + "GIT_COMMITTER_EMAIL": "dev@example.com", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + } + p = mock.patch.dict(os.environ, ident) + p.start() + self.addCleanup(p.stop) + origin, self.dev, self.work = ( + os.path.join(tmp.name, d) for d in ("origin", "dev", "work") + ) + rm.git(tmp.name, "init", "-q", "--bare", "-b", "main", origin) + rm.git(tmp.name, "clone", "-q", origin, self.dev) + self.commit(self.PATH, "a\n", "base") + rm.git(self.dev, "push", "-q", "origin", "main") + rm.git(self.dev, "checkout", "-q", "-b", "release/1.0") + self.commit(self.PATH, "b\n", "line change") + rm.git(self.dev, "push", "-q", "origin", "release/1.0") + rm.git(self.dev, "checkout", "-q", "main") + self.fix = self.commit(self.PATH, "c\n", "fix") + rm.git(self.dev, "push", "-q", "origin", "main") + rm.git(tmp.name, "clone", "-q", origin, self.work) + with open(os.path.join(self.work, "state.json"), "w") as f: + f.write("{}\n") # relstate's output, untracked in the job's checkout + cwd = os.getcwd() + os.chdir(self.work) # backport.REPO is "." + self.addCleanup(os.chdir, cwd) + self.pushed = [] + for name, fake in ( + ("push_head", lambda *a: self.pushed.append(a) or ("failed", "stop")), + ("fix_labels", lambda *a: []), + ): + p = mock.patch.object(bp, name, fake) + p.start() + self.addCleanup(p.stop) + + def commit(self, path, text, msg): + os.makedirs(os.path.dirname(os.path.join(self.dev, path)), exist_ok=True) + with open(os.path.join(self.dev, path), "w") as f: + f.write(text) + rm.git(self.dev, "add", path) + rm.git(self.dev, "commit", "-q", "-m", msg) + return rm.git(self.dev, "rev-parse", "HEAD") + + def test_conflict_commit_has_no_untracked_file(self): + self.assertEqual( + bp.pick("o/r", self.fix, "release/1.0", "t", "#1"), ("failed", "stop") + ) + self.assertEqual(len(self.pushed), 1) + self.assertEqual( + rm.git(".", "show", "-z", "--name-only", "--format=", "HEAD") + .rstrip("\0") + .split("\0"), + [self.PATH], + ) + self.assertIn( + f"Conflicts:\n\t{self.PATH}\n", rm.git(".", "log", "-1", "--format=%B") + "\n" + ) + + def test_pick_merged_by_another_run_after_this_one_fetched(self): + rm.git(self.dev, "checkout", "-q", "release/1.0") + rm.git( + self.dev, + "commit", + "-q", + "--allow-empty", + "-m", + f"fix\n\n(cherry picked from commit {self.fix})", + ) + rm.git(self.dev, "push", "-q", "origin", "release/1.0") + self.assertEqual( + bp.pick("o/r", self.fix, "release/1.0", "t", "#1"), + ("skipped", "already on branch"), + ) + self.assertEqual(self.pushed, []) + + +class PickGitPathWithSpace(PickGit): + """The same against a conflicted path with a space and a non-ASCII name (this repo has paths with spaces).""" + + PATH = "docker (deprecated)/é.txt" + + if __name__ == "__main__": unittest.main() diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index 59eb9d6963..c415fe8cdb 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -36,6 +36,22 @@ def test_prefix_optional(self): "superseded by #12", ) + def test_pick_trailer_dropped( + self, + ): # it would mark that commit declined in the decline commit's message + self.assertEqual( + bm.decline_reason( + [comment(f"Declined: dup of (cherry picked from commit {'a' * 40})")], + ME, + CLOSED, + ), + "dup of", + ) + nested = f"(cherry picked from (cherry picked from commit {'b' * 40})commit {'a' * 40})" + self.assertEqual( + bm.decline_reason([comment(f"Declined: dup {nested}")], ME, CLOSED), "dup" + ) + def test_none(self): self.assertIsNone(bm.decline_reason([], ME, CLOSED)) self.assertIsNone(bm.decline_reason([comment("Declined: ")], ME, CLOSED)) @@ -390,10 +406,14 @@ def test_base_modified_three_times_raises(self): self.run_squash([self.MOVED] * 3) def test_pending_check_is_not_retried(self): - self.assertEqual( - self.run_squash([RuntimeError("Required status check is expected")]), - (False, 1, []), - ) + with mock.patch("builtins.print") as out: + self.assertEqual( + self.run_squash([RuntimeError("Required status check is expected")]), + (False, 1, []), + ) + out.assert_called_once_with( + "#4: Required status check is expected" + ) # GitHub's words reach the log def test_other_error_is_not_retried(self): with self.assertRaisesRegex(RuntimeError, "HTTP 409"): diff --git a/.github/release/test_convgate.py b/.github/release/test_convgate.py index f1c0559653..5c3de38835 100644 --- a/.github/release/test_convgate.py +++ b/.github/release/test_convgate.py @@ -37,8 +37,8 @@ def setUp(self): self.change(f"work on {line}", f"release/{line}") self.c = self.change("C: the fix", "main") self.pick(self.c, "release/1.5") - self.push("cloud/release", "v1.5.0-rc.1", "release/1.5") - self.push("cloud/release", "v1.6.0-rc.1", "release/1.6") + self.push("cloud/backend", "v1.5.0-rc.1", "release/1.5") + self.push("cloud/backend", "v1.6.0-rc.1", "release/1.6") def g(self, *args): return rm.git( @@ -175,7 +175,7 @@ def test_2f_pick_after_decline_past_ga_still_owed(self): """Newer is GA at a tag without the fix; its decline was undone by a later pick: older's promote waits for newer's next patch instead of reading the stale decline as converged. """ - self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"]["v1.6.0-rc.1"] self.g("tag", "v1.6.0", "v1.6.0-rc.1") self.decline(self.c, "release/1.6") self.pick(self.c, "release/1.6") @@ -337,7 +337,7 @@ def test_7c_no_pr_fallback_warns(self): self.assertIn(msg, out) def test_7d_on_newer_tip_not_shipped(self): - self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"][ + self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"][ "v1.6.0-rc.1" ] # 1.6 GA at v1.6.0 self.g("tag", "v1.6.0", "v1.6.0-rc.1") @@ -365,11 +365,11 @@ def test_7e_on_promoted_tip_not_in_its_rc(self): def onprem(self): """On-prem RCs on both lines (the cloud RC commits), then on-prem Promote of 1.6 at v1.6.0, chart not published.""" for tag in ("v1.5.0-rc.1", "v1.6.0-rc.1"): - self.reg.setdefault("on-prem/release", {})[tag] = self.reg["cloud/release"][ + self.reg.setdefault("on-prem/backend", {})[tag] = self.reg["cloud/backend"][ tag ] self.g("tag", "v1.6.0", "v1.6.0-rc.1") - self.reg["on-prem/release"]["v1.6.0"] = self.reg["on-prem/release"]["v1.6.0-rc.1"] + self.reg["on-prem/backend"]["v1.6.0"] = self.reg["on-prem/backend"]["v1.6.0-rc.1"] def test_7f_promoted_not_published_read_at_its_tag(self): self.onprem() @@ -390,13 +390,13 @@ def test_7f_promoted_not_published_read_at_its_tag(self): def test_7g_patch_promoted_read_over_published(self): self.onprem() - self.reg["on-prem/chart"] = { - "v1.6.0": self.reg["on-prem/release"]["v1.6.0"] + self.reg["chart/unstract-platform"] = { + "v1.6.0": self.reg["on-prem/backend"]["v1.6.0"] } # Publish v1.6.0 self.pick(self.c, "release/1.6") self.g("tag", "v1.6.1-rc.1", "release/1.6") self.g("tag", "v1.6.1", "release/1.6") - self.reg["on-prem/release"]["v1.6.1-rc.1"] = self.reg["on-prem/release"][ + self.reg["on-prem/backend"]["v1.6.1-rc.1"] = self.reg["on-prem/backend"][ "v1.6.1" ] = { "digest": "sha256:61", @@ -517,7 +517,7 @@ def test_9e_rbo_window(self): self.assertNotIn(old[:12], out) self.assertNotIn(workflow[:12], out) # 1.6 GA for cloud: its GA tag's date is the window - self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"]["v1.6.0-rc.1"] with mock.patch.dict(os.environ, GIT_COMMITTER_DATE="2031-01-01T00:00:00Z"): self.g("tag", "-a", "v1.6.0", "-m", "promote", "v1.6.0-rc.1") later = self.rbo("after the GA", "2032-01-01T00:00:00Z") @@ -527,7 +527,7 @@ def test_9e_rbo_window(self): self.assertNotIn(new[:12], out) def test_9f_checks_name_lines_as_needed(self): - self.reg["cloud/release"]["v1.5.0"] = self.reg["cloud/release"][ + self.reg["cloud/backend"]["v1.5.0"] = self.reg["cloud/backend"][ "v1.5.0-rc.1" ] # 1.5 GA: stabilized self.g("tag", "v1.5.0", "v1.5.0-rc.1") @@ -559,7 +559,7 @@ def test_9g_count_line(self): "into `refs/tags/v1.6.0-rc.1`): 1 picks, 0 in the RC, 1 missing, 1 unclassifiable\n", out, ) - self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"]["v1.6.0-rc.1"] self.g( "tag", "v1.6.0", "release/1.6" ) # 1.6 GA at a tag with the pick: named by its tag @@ -575,7 +575,7 @@ def test_9g_count_line(self): self.checks(out)[2], ) self.push( - "cloud/release", "v1.6.1-rc.1", "release/1.6" + "cloud/backend", "v1.6.1-rc.1", "release/1.6" ) # a patch RC of a GA line: "the RC" again _, out = self.run_gate("release/1.6", promote_ref="v1.6.1-rc.1") self.assertIn( @@ -647,14 +647,14 @@ def test_9d_main_blocks_on_promote_order(self): ) def test_3_scope_is_the_target(self): - self.push("on-prem/release", "v1.5.0-rc.2", "release/1.5") + self.push("on-prem/backend", "v1.5.0-rc.2", "release/1.5") code, out = self.run_gate("release/1.5", "on-prem") self.assertEqual(code, 0) self.assertIn("scanned branches: none", out) self.assertEqual(self.run_gate("release/1.5", "cloud")[0], 2) def test_4_ga_branch_scanned_at_its_ga_tag(self): - self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"][ + self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"][ "v1.6.0-rc.1" ] # promote, same digest self.g("tag", "v1.6.0", "v1.6.0-rc.1") @@ -667,7 +667,7 @@ def test_4_ga_branch_scanned_at_its_ga_tag(self): self.assertEqual(code, 0, out) def test_5_plain_fix_owed_to_stabilized(self): - self.reg["cloud/release"]["v1.6.0"] = self.reg["cloud/release"]["v1.6.0-rc.1"] + self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"]["v1.6.0-rc.1"] self.g("tag", "v1.6.0", "v1.6.0-rc.1") # 1.6 GA = stabilized for cloud code, out = self.run_gate("release/1.5") self.assertEqual(code, 2, out) @@ -835,7 +835,7 @@ def repin(self, line, pin, rc): ) self.g(self.cloud, "tag", rc) self.g(self.cloud, "checkout", "-q", "main") - reg = self.reg.setdefault("cloud/release", {}) + reg = self.reg.setdefault("cloud/backend", {}) reg[rc] = { "digest": f"sha256:{len(reg):064x}", "pushed_at": "2026-09-28T00:00:00Z", diff --git a/.github/release/test_links.py b/.github/release/test_links.py index 42d2afdf8b..1707e6edde 100644 --- a/.github/release/test_links.py +++ b/.github/release/test_links.py @@ -7,6 +7,7 @@ ACTIONS = { "GITHUB_SERVER_URL": "https://github.com", "GITHUB_REPOSITORY": "Zipstack/example", + "RELSTATE_AR_ON_PREM": "projects/p/locations/l/repositories/r", } WEB = "https://github.com/Zipstack/example" SHA = "6635951" + "0" * 33 @@ -24,8 +25,7 @@ def test_actions(self): self.assertEqual(rm.run_link(9), f"[run]({WEB}/actions/runs/9)") self.assertEqual( rm.image_link("on-prem", "v0.192.0-rc.1"), - "[`on-prem/release:v0.192.0-rc.1`](https://github.com/orgs/Zipstack/packages/container/" - "package/example%2Fon-prem%2Frelease)", + "[`on-prem:v0.192.0-rc.1`](https://console.cloud.google.com/artifacts/docker/p/l/r)", ) def test_plain_outside_actions(self): @@ -35,7 +35,7 @@ def test_plain_outside_actions(self): self.assertEqual(rm.tag_link("v0.192.0-rc.1"), "`v0.192.0-rc.1`") self.assertEqual(rm.run_link(9), "run 9") self.assertEqual( - rm.image_link("cloud", "v0.192.0-rc.1"), "`cloud/release:v0.192.0-rc.1`" + rm.image_link("cloud", "v0.192.0-rc.1"), "`cloud:v0.192.0-rc.1`" ) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 1e97980856..0a20d060c9 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -346,6 +346,105 @@ def test_retire(self): ) self.assertEqual(len(self.remote("tag", "-l", "retired/1.7-*").split()), 2) + def test_retire_keeps_a_branch_pushed_after_its_tip_was_read(self): + self.quiet(oss.cut, self.oss, "1.7") + self.clone() + self.g(self.dev, "fetch", "-q", "origin") # a commit lands after this run fetched + self.g(self.dev, "checkout", "-q", "-B", "release/1.7", "origin/release/1.7") + self.g(self.dev, "commit", "-q", "--allow-empty", "-m", "late fix") + self.g(self.dev, "push", "-q", "origin", "release/1.7") + late = self.g(self.dev, "rev-parse", "HEAD") + self.g(self.dev, "checkout", "-q", "main") + with self.assertRaises(RuntimeError): + oss.retire(self.oss, "release/1.7", "x", dry_run=False) + self.assertEqual(self.remote("rev-parse", "release/1.7"), late) + self.assertEqual(self.remote("tag", "-l", "ee-v1.7.*"), "ee-v1.7.0-rc.1") + self.assertEqual(self.remote("tag", "-l", "retired/*"), "") + self.assertEqual( + self.g(self.oss, "tag", "-l", "retired/*"), "" + ) # nor left in the clone + + def test_retire_rerun_in_the_same_clone_archives_the_tip(self): + self.quiet(oss.cut, self.oss, "1.7") + self.clone() + tip = self.remote("rev-parse", "release/1.7") + push, calls = oss._push, [] + + def flaky( + *a, + ): # the first push fails (network, a remote hook), the re-run's goes through + calls.append(a) + if len(calls) == 1: + raise RuntimeError("push failed") + return push(*a) + + with mock.patch.object(oss, "_push", flaky): + with self.assertRaises(RuntimeError): + oss.retire(self.oss, "release/1.7", "x", dry_run=False) + oss.retire(self.oss, "release/1.7", "x", dry_run=False) + self.assertEqual( + self.remote("rev-parse", f"retired/1.7-{tip[:7]}^{{commit}}"), tip + ) + self.assertEqual(self.remote("branch", "-l", "release/1.7"), "") + + def test_retire_pushes_an_archive_tag_left_in_the_clone(self): + self.quiet(oss.cut, self.oss, "1.7") + self.clone() + tip = self.remote("rev-parse", "release/1.7") + oss._tag(self.oss, f"retired/1.7-{tip[:7]}", tip, "a run killed before its push") + oss.retire(self.oss, "release/1.7", "x", dry_run=False) + self.assertEqual( + self.remote("rev-parse", f"retired/1.7-{tip[:7]}^{{commit}}"), tip + ) + + def test_rc_refuses_a_branch_deleted_after_its_tip_was_read(self): + self.quiet(oss.cut, self.oss, "1.7") + self.work("release/1.7", "oss fix") # an untagged tip, so rc bumps + self.clone() + self.remote( + "update-ref", "-d", "refs/heads/release/1.7" + ) # a retire lands after this run fetched + with self.assertRaises(RuntimeError): + self.quiet(oss.rc, self.oss, "release/1.7") + self.assertEqual(self.remote("branch", "-l", "release/1.7"), "") + + def test_cut_refuses_a_branch_created_after_it_fetched(self): + self.clone() + main = self.remote("rev-parse", "main") + self.remote( + "update-ref", "refs/heads/release/1.7", main + ) # another run cut the line after this run fetched + with self.assertRaises(RuntimeError): + self.quiet(oss.cut, self.oss, "1.7") + self.assertEqual(self.remote("rev-parse", "release/1.7"), main) + self.assertEqual( + rm.git(self.oss, "tag", "-l", "ee-v1.7.0-rc.*"), "" + ) # nor left in the clone, where a re-run would read the minor as used + + def test_promote_rerun_in_the_same_clone_pushes_the_tag(self): + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + push, calls = oss._push, [] + + def flaky(*a): # the first push fails, the re-run's goes through + calls.append(a) + if len(calls) == 1: + raise RuntimeError("push failed") + return push(*a) + + with mock.patch.object(oss, "_push", flaky): + with self.assertRaises(RuntimeError): + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + self.assertEqual(rm.git(self.oss, "tag", "-l", "ee-v1.7.0"), "") + self.assertEqual( + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1"), "ee-v1.7.0" + ) + self.assertEqual(len(calls), 2) + self.assertEqual( + self.remote("rev-parse", "ee-v1.7.0^{commit}"), + self.remote("rev-parse", "ee-v1.7.0-rc.1^{commit}"), + ) + def test_retire_refuses_a_line_moved_after_its_archive(self): self.quiet(oss.cut, self.oss, "1.7") tip = self.remote("rev-parse", "release/1.7") diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index dad4139fc6..a57c6986bb 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -1,11 +1,17 @@ -"""Synthetic-repo tests for relstate (derive 1-11, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37). Run: python test_relstate.py""" +"""Synthetic-repo tests for relstate (derive 1-11, on-prem newest six 7b, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ticketless titles 18c, Release-branch-only key 18d, release-branch-only label 18e, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, no order block on a GA line 23b, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37, Artifact Registry reads 38-41 and 39b). Run: python test_relstate.py. No workflow runs the .github/release suites yet: run all six locally before +changing this directory. +""" import contextlib import datetime as dt import io +import json import os import tempfile import unittest +import unittest.mock +import urllib.error +import urllib.parse os.environ["GITHUB_REPOSITORY"] = ( "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets @@ -70,35 +76,35 @@ def ga(self, artifact, tag): # Scenarios build on each other as the table does. def s1(self): self.cut("1.5") - self.push("cloud/release", "v1.5.0-rc.1", git_ref="release/1.5") + self.push("cloud/backend", "v1.5.0-rc.1", git_ref="release/1.5") def s2(self): self.s1() - self.push("cloud/release", "v1.5.0", git_ref="release/1.5") + self.push("cloud/backend", "v1.5.0", git_ref="release/1.5") def s3(self): self.s2() - self.push("on-prem/release", "v1.5.0-rc.3", git_ref="release/1.5") + self.push("on-prem/backend", "v1.5.0-rc.3", git_ref="release/1.5") def s4(self, chart_days=10): self.s3() - self.push("on-prem/chart", "v1.5.1", days=chart_days) + self.push("chart/unstract-platform", "v1.5.1", days=chart_days) def s6(self, chart_days=10): self.s4(chart_days) self.cut("1.6") - self.push("cloud/release", "v1.6.0-rc.1", git_ref="release/1.6") - self.push("cloud/release", "v1.6.0", git_ref="release/1.6") + self.push("cloud/backend", "v1.6.0-rc.1", git_ref="release/1.6") + self.push("cloud/backend", "v1.6.0", git_ref="release/1.6") def cloud_after_2(self, state): - ga = self.ga("cloud/release", "v1.5.0") + ga = self.ga("cloud/backend", "v1.5.0") return {"state": state, "rcs": ["v1.5.0-rc.1"], "ga": ga, "promoted": ga} def onprem(self, state, chart=True): return { "state": state, "rcs": ["v1.5.0-rc.3"], - "ga": self.ga("on-prem/chart", "v1.5.1") if chart else None, + "ga": self.ga("chart/unstract-platform", "v1.5.1") if chart else None, "promoted": None, } @@ -259,10 +265,10 @@ def test_7_retire(self): def test_7b_onprem_newest_six(self): for y in range(1, 9): # 1.1 .. 1.8, each with an on-prem chart inside the window self.cut(f"1.{y}") - self.push("on-prem/release", f"v1.{y}.0-rc.1", git_ref=f"release/1.{y}") - self.push("on-prem/chart", f"v1.{y}.0", days=10) + self.push("on-prem/backend", f"v1.{y}.0-rc.1", git_ref=f"release/1.{y}") + self.push("chart/unstract-platform", f"v1.{y}.0", days=10) self.cut("1.9") # no on-prem GA: does not count toward the six - self.push("on-prem/release", "v1.9.0-rc.1", git_ref="release/1.9") + self.push("on-prem/backend", "v1.9.0-rc.1", git_ref="release/1.9") st = self.state() self.assertEqual( st["supported"]["on-prem"], [f"release/1.{y}" for y in range(9, 2, -1)] @@ -291,7 +297,7 @@ def test_8_cut_no_artifacts(self): def test_9_tag_not_on_branch(self): self.s1() - self.push("on-prem/release", "v1.5.0-rc.9", git_ref=self.commit("main-only")) + self.push("on-prem/backend", "v1.5.0-rc.9", git_ref=self.commit("main-only")) st = self.state() self.assertEqual(self.branch(st, "release/1.5")["enabled"], ["cloud"]) self.assertEqual( @@ -299,7 +305,7 @@ def test_9_tag_not_on_branch(self): [ { "target": "on-prem", - "artifact": "on-prem/release", + "artifact": "on-prem/backend", "tag": "v1.5.0-rc.9", "branch": "release/1.5", "reason": "commit not on branch", @@ -336,13 +342,13 @@ def test_13_retired_lines_are_history_not_anomalies(self): self.s4() # only release/1.5 is live retired = self.commit("1.6 work") self.push( - "cloud/release", "v1.6.0-rc.1", git_ref=retired + "cloud/backend", "v1.6.0-rc.1", git_ref=retired ) # promoted, published, then retired: no branch - self.push("cloud/release", "v1.6.0", git_ref=retired) - self.push("on-prem/release", "v1.6.0") - self.push("on-prem/chart", "v1.6.0", days=400) + self.push("cloud/backend", "v1.6.0", git_ref=retired) + self.push("on-prem/backend", "v1.6.0") + self.push("chart/unstract-platform", "v1.6.0", days=400) self.push( - "cloud/release", "v1.9.0-rc.1", git_ref=self.commit("never promoted") + "cloud/backend", "v1.9.0-rc.1", git_ref=self.commit("never promoted") ) # still an anomaly st = self.state() self.assertEqual( @@ -350,7 +356,7 @@ def test_13_retired_lines_are_history_not_anomalies(self): [ { "target": "cloud", - "artifact": "cloud/release", + "artifact": "cloud/backend", "tag": "v1.9.0-rc.1", "branch": "release/1.9", "reason": "no branch", @@ -418,7 +424,7 @@ def test_22_promote_blockers(self): msg = "release/{} is not GA for {} yet: promote and publish it first, retire it, or force with a reason" self.s3() # 1.5: cloud GA, on-prem RC only self.cut("1.6") - self.push("cloud/release", "v1.6.0-rc.1", git_ref="release/1.6") + self.push("cloud/backend", "v1.6.0-rc.1", git_ref="release/1.6") self.cut("1.7") # nothing built yet st = self.state() self.assertEqual( @@ -444,7 +450,7 @@ def test_22_promote_blockers(self): rs.promote_blockers(st, "release/1.8", "cloud"), [msg.format("1.6", "cloud")] ) # (d) 1.5 publishes on-prem (chart push): 1.6 on-prem is no longer blocked - self.push("on-prem/chart", "v1.5.1") + self.push("chart/unstract-platform", "v1.5.1") self.assertEqual(rs.promote_blockers(self.state(), "release/1.6", "on-prem"), []) # (e) a retired older line (branch gone) never blocks self.g("branch", "-D", "release/1.6") @@ -454,7 +460,7 @@ def test_23_enable_blockers(self): """A target is enabled only on a line newer than every line already GA for it.""" msg = "release/{} is older than release/{}, already GA for {}: enable {} on a newer line instead" self.cut("1.4") - self.push("cloud/release", "v1.4.0-rc.1", git_ref="release/1.4") + self.push("cloud/backend", "v1.4.0-rc.1", git_ref="release/1.4") self.s6() # 1.5: cloud GA, on-prem GA (chart v1.5.1); 1.6: cloud GA, no on-prem st = self.state() # (a) an older line enabling on-prem behind a line GA for on-prem is refused @@ -473,8 +479,8 @@ def test_23b_no_order_block_on_a_ga_line(self): line already GA for a target is never order-blocked. Without the enable rule it can be. """ rc, ga = ( - {"cloud": ("cloud/release", ".0"), "on-prem": ("on-prem/release", ".1")}, - {"cloud": "cloud/release", "on-prem": "on-prem/chart"}, + {"cloud": ("cloud/backend", ".0"), "on-prem": ("on-prem/backend", ".1")}, + {"cloud": "cloud/backend", "on-prem": "chart/unstract-platform"}, ) refused = [] @@ -537,7 +543,7 @@ def check(): ], ) # enabling on-prem on 1.4 behind GA lines anyway order-blocks every newer GA line's patch - self.push("on-prem/release", "v1.4.1-rc.1", git_ref="release/1.4") + self.push("on-prem/backend", "v1.4.1-rc.1", git_ref="release/1.4") self.assertTrue(rs.promote_blockers(self.state(), "release/1.6", "on-prem")) def chart(self, ver, branch, msg, email="dev@example.com"): @@ -578,7 +584,7 @@ def test_24_already_bumped(self): def test_25_promoted_from_build_artifact(self): self.s3() self.push( - "on-prem/release", "v1.5.1", git_ref="release/1.5" + "on-prem/backend", "v1.5.1", git_ref="release/1.5" ) # Promote on-prem, chart not published onprem = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] self.assertEqual( @@ -587,19 +593,19 @@ def test_25_promoted_from_build_artifact(self): "state": "stabilizing", "rcs": ["v1.5.0-rc.3"], "ga": None, - "promoted": self.ga("on-prem/release", "v1.5.1"), + "promoted": self.ga("on-prem/backend", "v1.5.1"), }, ) self.push( - "on-prem/chart", "v1.5.1" + "chart/unstract-platform", "v1.5.1" ) # Publish: GA and state move, promoted does not onprem = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] self.assertEqual( (onprem["state"], onprem["ga"], onprem["promoted"]), ( "stabilized", - self.ga("on-prem/chart", "v1.5.1"), - self.ga("on-prem/release", "v1.5.1"), + self.ga("chart/unstract-platform", "v1.5.1"), + self.ga("on-prem/backend", "v1.5.1"), ), ) @@ -655,20 +661,20 @@ def built_by_mistake(self): "pin", ) self.g("checkout", "-q", "main") - self.push("cloud/release", "v1.9.0-rc.1", git_ref="release/1.9") - self.push("on-prem/release", "v1.9.0-rc.1") + self.push("cloud/backend", "v1.9.0-rc.1", git_ref="release/1.9") + self.push("on-prem/backend", "v1.9.0-rc.1") self.push( - "cloud/release", "v1.5.0" + "cloud/backend", "v1.5.0" ) # other versions keep the cloud package non-empty def test_16_abandon_plan_built_by_mistake(self): self.built_by_mistake() - self.push("cloud/release", "v1.9.0-rc.2") - self.reg["cloud/release"]["edge"] = self.reg["cloud/release"][ + self.push("cloud/backend", "v1.9.0-rc.2") + self.reg["cloud/backend"]["edge"] = self.reg["cloud/backend"][ "v1.9.0-rc.2" ] # mixed tags: kept plan = rs.abandon_plan(self.repo, "release/1.9", lambda a: self.reg.get(a, {})) - rc1 = self.reg["cloud/release"]["v1.9.0-rc.1"]["digest"] + rc1 = self.reg["cloud/backend"]["v1.9.0-rc.1"]["digest"] self.assertEqual( plan, { @@ -676,13 +682,13 @@ def test_16_abandon_plan_built_by_mistake(self): "used": [], "rc_tags": ["v1.9.0-rc.1"], "versions": { - "cloud/release": [ + "cloud/backend": [ {"id": None, "digest": rc1, "tags": ["v1.9.0-rc.1"]} ] }, - "kept": ["on-prem/release:v1.9.0-rc.1"], + "kept": ["on-prem/backend:v1.9.0-rc.1"], }, - ) # last version of on-prem/release + ) # last version of on-prem/backend self.g("branch", "release/1.8", "main") # never built: nothing to delete self.assertEqual( rs.abandon_plan(self.repo, "release/1.8", lambda a: {})["versions"], {} @@ -693,17 +699,17 @@ def test_17_abandon_plan_refusals(self): plan = lambda: rs.abandon_plan( self.repo, "release/1.9", lambda a: self.reg.get(a, {}) ) - self.push("on-prem/chart", "v1.9.0") - self.assertEqual(plan()["ga"], ["on-prem/chart:v1.9.0"]) + self.push("chart/unstract-platform", "v1.9.0") + self.assertEqual(plan()["ga"], ["chart/unstract-platform:v1.9.0"]) self.g("tag", "v1.9.0", "release/1.9") - self.assertEqual(plan()["ga"], ["v1.9.0", "on-prem/chart:v1.9.0"]) + self.assertEqual(plan()["ga"], ["v1.9.0", "chart/unstract-platform:v1.9.0"]) human = self.commit("human RBO-less work", "release/1.9") self.assertEqual(plan()["used"], [human[:12]]) def test_26_oss_repo_config(self): # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), # supported while its branch exists. Plain v* tags are community releases from main and never count. - self.addCleanup(rs.use, rs.PKG_PREFIX) + self.addCleanup(rs.use, rs.SLUG) rs.use("unstract") self.state = lambda: rs.derive( self.repo, NOW, lambda a: self.fail(f"registry read of {a}") @@ -801,10 +807,10 @@ def test_29_retire_line(self): "cloud is production (newest cloud GA); it retires once a newer line is promoted", ) self.cut("1.6") - self.push("cloud/release", "v1.6.0-rc.1", git_ref="release/1.6") - self.push("cloud/release", "v1.6.0", git_ref="release/1.6") + self.push("cloud/backend", "v1.6.0-rc.1", git_ref="release/1.6") + self.push("cloud/backend", "v1.6.0", git_ref="release/1.6") self.assertIsNone(refusal()) # cloud past-eos, on-prem never promoted: may go - self.push("on-prem/chart", "v1.5.1") # on-prem chart in its window + self.push("chart/unstract-platform", "v1.5.1") # on-prem chart in its window self.assertEqual( refusal(), "on-prem still supported (stabilized); force to retire it inside the window", @@ -855,8 +861,8 @@ def test_30_rc_version_skips_registry_rcs(self): rm.rc_version(self.repo, "release/1.5", taken=taken), "v1.5.0-rc.3" ) # relstate allocates past every target's registry RCs. - self.push("cloud/release", "v1.5.0-rc.3") - self.push("on-prem/release", "v1.5.0-rc.4") + self.push("cloud/backend", "v1.5.0-rc.3") + self.push("on-prem/backend", "v1.5.0-rc.4") self.assertEqual( self.branch(self.state(), "release/1.5")["next_rc"], "v1.5.0-rc.5" ) @@ -866,7 +872,7 @@ def test_31_reused_minor_skips_its_kept_rc(self): # version). The minor is cut again, and its first RC is past the kept one. self.cut("1.6") tip = self.cut("1.7") - self.push("on-prem/release", "v1.7.0-rc.1") + self.push("on-prem/backend", "v1.7.0-rc.1") self.g("tag", f"retired/1.7-{tip[:7]}", tip) self.g("branch", "-D", "release/1.7") self.assertEqual(self.state()["next_minor"], "1.7") @@ -1070,16 +1076,177 @@ def test_18c_ticketless_title_same_class(self): self.assertEqual(c("[MISC] x", ["fix/needs-backport"]), "fix") self.assertEqual(c("[FEAT] x", ["fix/security"]), "security") self.assertEqual(c('Revert "[FIX] x"'), "fix") - for title in ("[fix] x", "[FIX]x", " [FIX] x", "UN-1[FIX] x", "un-1 [FIX] x"): + self.assertEqual(c(" UN-1 [FIX] x"), "fix") # GitHub keeps a typo'd leading space + # The revert button copies the title, leading space included. + self.assertEqual(c('Revert " UN-1 [FIX] x"'), "fix") + for title in ("[fix] x", "[FIX]x", "UN-1[FIX] x", "un-1 [FIX] x"): with self.assertRaises(ValueError, msg=title): c(title) + def test_18e_release_only_label_refused_off_release(self): + f = rm.release_only_refusal + self.assertIn( + "for PRs into a release/X.Y branch", f("main", ["release-branch-only"]) + ) + self.assertIsNone(f("release/0.192", ["release-branch-only"])) + self.assertIsNone(f("main", ["fix/critical"])) + def test_18d_rbo_key_ticket_else_pr(self): self.assertEqual(rm.rbo_key("UN-1 [FIX] x", "45"), "UN-1") self.assertEqual(rm.rbo_key("[FIX] x", "45"), "#45") + self.assertEqual(rm.rbo_key(" UN-1 [FIX] x", "45"), "UN-1") self.assertEqual(rm.rbo_key("no prefix", "45"), "#45") self.assertEqual(rm.RBO_RE.findall("Release-branch-only: #45"), ["#45"]) +class ArtifactRegistry(unittest.TestCase): + """registry_tags over the Artifact Registry REST API. The response shape (versions[].name, createTime, + relatedTags[].name, nextPageToken) follows the API reference for projects.locations.repositories.packages + .versions.list with view=FULL; it was not recorded from a live call. + """ + + REPO = "projects/p/locations/us-central1/repositories/unstract" + + def setUp(self): + env = { + "RELSTATE_AR_TOKEN": "t", + "RELSTATE_AR_CLOUD": self.REPO, + "RELSTATE_AR_ON_PREM": "projects/p/locations/us-central1/repositories/unstract-on-prem", + } + patcher = unittest.mock.patch.dict(os.environ, env) + patcher.start() + self.addCleanup(patcher.stop) + self.urls, self.pages = [], {} + urlopen = unittest.mock.patch.object(rs.urllib.request, "urlopen", self.urlopen) + urlopen.start() + self.addCleanup(urlopen.stop) + + def urlopen(self, req, timeout): + self.urls.append(req.full_url) + self.assertEqual(req.get_header("Authorization"), "Bearer t") + path, _, query = req.full_url.partition("?") + key = (path, urllib.parse.parse_qs(query).get("pageToken", [""])[0]) + page = self.pages.get(key, 404) + if isinstance(page, int): + raise urllib.error.HTTPError( + req.full_url, page, "err", {}, io.BytesIO(b"denied") + ) + return io.BytesIO(json.dumps(page).encode()) + + def version(self, package, digest, created, *tags): + name = f"{self.REPO}/packages/{package}/versions/sha256:{digest}" + return { + "name": name, + "createTime": created, + "updateTime": "2030-01-01T00:00:00Z", # differs from createTime: pushed_at must not read it + "relatedTags": [ + {"name": f"{self.REPO}/packages/{package}/tags/{t}", "version": name} + for t in tags + ], + } + + def url(self, package, kind="unstract"): + return f"{rs.AR_API}/projects/p/locations/us-central1/repositories/{kind}/packages/{package}/versions" + + def test_38_tags_across_pages(self): + self.pages[(self.url("backend"), "")] = { + "versions": [ + self.version( + "backend", + "aa", + "2026-10-01T10:00:00.123456789Z", + "v0.192.0-rc.1", + "v0.192.0", + ) + ], + "nextPageToken": "n2", + } + self.pages[(self.url("backend"), "n2")] = { + "versions": [ + self.version("backend", "bb", "2026-09-01T10:00:00Z", "v0.191.0"), + self.version("backend", "cc", "2026-08-01T10:00:00.5Z"), + ] + } + got = rs.registry_tags("cloud/backend") + v1 = f"{self.REPO}/packages/backend/versions/sha256:aa" + self.assertEqual( + got["v0.192.0-rc.1"], + {"id": v1, "digest": "sha256:aa", "pushed_at": "2026-10-01T10:00:00Z"}, + ) + self.assertEqual(got["v0.192.0"]["digest"], "sha256:aa") + self.assertEqual(got["v0.191.0"]["pushed_at"], "2026-09-01T10:00:00Z") + self.assertEqual( + set(got), {"v0.192.0-rc.1", "v0.192.0", "v0.191.0"} + ) # an untagged version adds nothing + self.assertIn("view=FULL", self.urls[0]) + + def test_39_404_is_empty_only_when_repository_reads(self): + onprem = "projects/p/locations/us-central1/repositories/unstract-on-prem" + self.pages[(f"{rs.AR_API}/{onprem}", "")] = {"name": onprem} + self.assertEqual(rs.registry_tags("on-prem/backend"), {}) + self.assertEqual( + self.urls[-1], f"{rs.AR_API}/{onprem}" + ) # its own repository, not cloud's + self.pages[(f"{rs.AR_API}/{self.REPO}", "")] = {"name": self.REPO} + self.pages[(f"{rs.AR_API}/{onprem}", "")] = 404 + with self.assertRaisesRegex( + RuntimeError, f"404 and repository {onprem} not found" + ): + rs.registry_tags("on-prem/backend") + self.pages[(f"{rs.AR_API}/{onprem}", "")] = 403 + with self.assertRaisesRegex(RuntimeError, "HTTP 403: denied"): + rs.registry_tags("on-prem/backend") + + def test_39b_network_json_and_later_page_errors_raise(self): + self.pages[(self.url("backend"), "")] = {"versions": [], "nextPageToken": "n2"} + with self.assertRaisesRegex(RuntimeError, "HTTP 404 on page n2"): # page n2: 404 + rs.registry_tags("cloud/backend") + for err in ( # what urlopen and the body read raise + urllib.error.URLError("dns"), + TimeoutError("timed out"), + rs.http.client.IncompleteRead(b""), + ConnectionResetError(), + ): + with ( + unittest.mock.patch.object(rs.urllib.request, "urlopen", side_effect=err), + self.assertRaisesRegex( + RuntimeError, f"registry read cloud/backend: {type(err).__name__}" + ), + ): + rs.registry_tags("cloud/backend") + with ( + unittest.mock.patch.object( + rs.urllib.request, "urlopen", return_value=io.BytesIO(b"") + ), + self.assertRaisesRegex( + RuntimeError, "registry read cloud/backend: JSONDecodeError" + ), + ): + rs.registry_tags("cloud/backend") + + def test_40_other_http_errors_raise(self): + self.pages[(self.url("backend"), "")] = 403 + with self.assertRaisesRegex(RuntimeError, "HTTP 403: denied"): + rs.registry_tags("cloud/backend") + cut = urllib.error.HTTPError("u", 503, "err", {}, None) + cut.read = unittest.mock.Mock(side_effect=rs.http.client.IncompleteRead(b"")) + with ( + unittest.mock.patch.object(rs.urllib.request, "urlopen", side_effect=cut), + self.assertRaisesRegex( + RuntimeError, "registry read cloud/backend: HTTP 503: IncompleteRead" + ), + ): # the error body itself cut off + rs.registry_tags("cloud/backend") + + def test_41_unset_location_or_token_raises(self): + os.environ.pop("RELSTATE_AR_CHART", None) + with self.assertRaisesRegex(RuntimeError, "set RELSTATE_AR_CHART"): + rs.registry_tags("chart/unstract-platform") + del os.environ["RELSTATE_AR_TOKEN"] + with self.assertRaisesRegex(RuntimeError, "set RELSTATE_AR_TOKEN"): + rs.registry_tags("cloud/backend") + self.assertEqual(self.urls, []) + + if __name__ == "__main__": unittest.main() From 4910d26f28f5715bd555cb5c5246b04bf8bee40e Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 12:27:35 +0530 Subject: [PATCH 04/37] chore: leave the release-branch-ci failure alert as a TODO Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/release-branch-ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-branch-ci.yaml b/.github/workflows/release-branch-ci.yaml index f17eaba860..214f8971bf 100644 --- a/.github/workflows/release-branch-ci.yaml +++ b/.github/workflows/release-branch-ci.yaml @@ -5,7 +5,7 @@ name: Release branch CI # On a PR into release/** this is the required check `check` over the PR's own commits, so a human change # is checked before it lands. A backport/X.Y/ PR is judged as its squash: only its first commit (the pick, whose # trailer the squash message keeps) is classified, so a plain resolution commit passes; conflict markers in the -# head's tree fail it. The push run stays as the backstop for bot pushes and alerts on failure. +# head's tree fail it. The push run stays as the backstop for bot pushes. # A pull_request run uses the workflow file from the PR head: a branch cut before this trigger existed gets no PR # check, and a PR can edit this file. Job `check` runs there; it holds no secret. # A PR labelled `release-branch-only` passes with commits of any kind. On the label, job `trailer` pushes one @@ -101,6 +101,8 @@ jobs: GH_TOKEN: ${{ github.token }} run: python3 .relmgr/.github/release/relstate.py | jq --arg b "$BRANCH" '.branches[] | select(.branch == $b)' + # TODO: alert on a failed push run once a release alert channel and webhook secret exist. + # On the `release-branch-only` label: one empty bot commit carrying `Release-branch-only: ` (ticket from # the title, else `#`), so the squash message has the line. A commit already carrying it -> nothing pushed. The push fires # `synchronize`, so `check` runs once more. A fork head cannot take a bot push: fail and say so. From 2a4541306d0bd4e55ca95803326140a10366f269 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 12:59:21 +0530 Subject: [PATCH 05/37] fix: tighten the release workflows after self-review round 1 - release-branch-ci: the trailer job runs only on an open PR and refuses a head named main, release/* or backport/*. Merge note no longer suggests merging when the commit check did not finish. - lint-pr: call relmgr.release_only_refusal instead of an inline copy; the header says a fix/ label waives the [TYPE] prefix. - backport: drop a stale PYTHONDONTWRITEBYTECODE and a comment naming workflows this repo does not have. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/backport.yaml | 4 +--- .github/workflows/lint-pr.yaml | 8 +++----- .github/workflows/release-branch-ci.yaml | 10 ++++++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index ae97a4427b..0d4fc14b03 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -22,7 +22,7 @@ run-name: >- # pull_request_target runs the base line's copy of this file, so it fires only on lines that carry it. Safe: # the job checks out main and never runs head code; backport_merge.py refuses a head that is not # backport/X.Y/ into release/X.Y (an expression cannot match that pattern, so the job filter is coarser). A failed record run is recovered with "Re-run jobs". -# Scripts run from main's tree, like Promote and Take RC build. +# Scripts run from main's tree. on: push: branches: [main] @@ -60,8 +60,6 @@ jobs: # deleted the head first, the loser can open a second pick PR for a fix already on the line. A squash after # the base moved is retried by backport_merge.squash. group: backport-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'rescan' }} - env: - PYTHONDONTWRITEBYTECODE: "1" # the conflict path runs `git add -A` steps: - name: Inputs if: github.event_name == 'workflow_dispatch' diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index 3d60a62542..6c6d4f4c6a 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -1,7 +1,7 @@ name: Lint PR # Every PR into main or a release branch carries a recognised `[TYPE]` title prefix, the ticket before it optional -# (`UN-123 [FIX] ...` or `[FIX] ...`, the same class either way), and its labels are +# (`UN-123 [FIX] ...` or `[FIX] ...`, the same class either way), unless a fix/ label gives its class; its labels are # at most one of fix/security, fix/critical plus an optional fix/needs-backport. # Release branches too, because the release changelog is built from their PR titles. A branch cut before this file # existed runs no check. Not a required check until a ruleset makes it one. @@ -50,10 +50,8 @@ jobs: title, labels = os.environ["TITLE"], json.loads(os.environ["LABELS"]) summary = open(os.environ["GITHUB_STEP_SUMMARY"], "a") try: - if "release-branch-only" in labels: - if not os.environ["BASE"].startswith("release/"): - raise ValueError("label `release-branch-only` is for PRs into a release/X.Y branch; remove it " - "here (a main change reaches release branches through Backport)") + if refusal := relmgr.release_only_refusal(os.environ["BASE"], labels): + raise ValueError(refusal) cls = relmgr.pr_fix_class(title, labels) or "none" except ValueError as e: msg = (f"{e}\nFix: title as `[TYPE] text` or `TICKET [TYPE] text` (e.g. `UN-123 [FIX] ...`); allowed labels are " diff --git a/.github/workflows/release-branch-ci.yaml b/.github/workflows/release-branch-ci.yaml index 214f8971bf..955e4254e2 100644 --- a/.github/workflows/release-branch-ci.yaml +++ b/.github/workflows/release-branch-ci.yaml @@ -13,7 +13,7 @@ name: Release branch CI # merge keeps it in the message, and the push run checks that merged commit like any other. It holds the bot token, # so it runs on pull_request_target: the base line's copy of this file, which a PR cannot edit. A line cut before # that trigger existed gets no bot commit; add the line to a commit yourself there. -# Label changes on a closed PR run nothing. +# Label changes on a closed PR run nothing. The bot never pushes to a head named main, release/* or backport/*. # No push builds an RC: a security or critical pick lands and waits for the RM's Take RC build, like any other pick. on: push: @@ -83,11 +83,14 @@ jobs: env: RBO_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'release-branch-only') }} INVALID: ${{ steps.classify.outputs.invalid || '0' }} + OUTCOME: ${{ steps.classify.outcome }} run: | if [ "$INVALID" != 0 ]; then hint="A commit here is not a pick from main. A change only for $BRANCH: add the \`release-branch-only\` label. A fix on main: cherry-pick it with \`git cherry-pick -x\`." echo "::notice::$hint" echo "$hint" >> "$GITHUB_STEP_SUMMARY" + elif [ "$OUTCOME" != success ]; then + echo "The commit check did not finish; see its log. Do not merge until it passes." >> "$GITHUB_STEP_SUMMARY" elif [ "$RBO_LABEL" = true ]; then echo "Squash and merge: the \`Release-branch-only: \` commit the bot added carries the line." >> "$GITHUB_STEP_SUMMARY" else @@ -109,7 +112,7 @@ jobs: # pull_request_target: this job is the base line's copy. The head is only data here: read without a token, its # files never imported, and the push ignores its git config. trailer: - if: github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name == 'release-branch-only' + if: github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name == 'release-branch-only' && github.event.pull_request.state == 'open' runs-on: ubuntu-latest timeout-minutes: 20 env: @@ -122,6 +125,9 @@ jobs: - name: Same-repo head run: | [ "$HEAD_REPO" = "$GITHUB_REPOSITORY" ] || { echo "::error::release-branch-only: the head is on $HEAD_REPO, a fork; the bot cannot push the \`Release-branch-only:\` commit there. Push the branch to $GITHUB_REPOSITORY and open the PR from it, or add the line to a commit yourself."; exit 1; } + case "$HEAD_REF" in + main | release/* | backport/*) echo "::error::release-branch-only: the head is $HEAD_REF; the bot pushes only to a topic branch. Add the line to a commit yourself."; exit 1 ;; + esac # The head is checked out without the bot token; only the push below gets it. The logic comes from main. # Run on the head's tip SHA, so a push to the head after the label makes the bot push fail, not skip a check. From b1b22fb2ca8d7691f99614382948cad3969124ff Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 14:21:58 +0530 Subject: [PATCH 06/37] fix: take the release tooling's final round of fixes; pass the App slug to record .github/release is copied unchanged from the enterprise repo's copy (tree ceebe2706f47). record() now identifies the bot's pick PRs by the App's slug, which the record job passes from the token step. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 31 ++++++++++--- .github/release/backport_merge.py | 17 ++++--- .github/release/convgate.py | 9 +++- .github/release/oss.py | 2 + .github/release/relmgr.py | 9 +++- .github/release/relstate.py | 26 +++++------ .github/release/test_backport.py | 62 +++++++++++++++++++++++++- .github/release/test_backport_merge.py | 23 +++++++--- .github/release/test_convgate.py | 7 +-- .github/release/test_relstate.py | 23 ++++++++++ .github/workflows/backport.yaml | 1 + 11 files changed, 173 insertions(+), 37 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index 949a7be6ae..695551de87 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -7,7 +7,8 @@ that no pick or decline trailer there names: plain fixes too on fix targets (held accept=OWED), security/critical only on stabilized ones (accept=HELD, held for the RM without --confirm-stabilized). A run cancelled by a newer push costs nothing: the next run rescans. The exception is a run stopped after pushing a pick head and before opening -its PR: after PICK_TIMEOUT that head fails later runs until a person deletes it (finish_open). +its PR, or whose `gh pr create` failed: after PICK_TIMEOUT that head fails later runs until a person deletes it +(finish_open). A decline carries the pick trailer, so a declined fix is never re-picked; an open conflict PR is left alone (`skipped: conflict PR pending`) until a person merges or closes it; an open clean pick PR whose merge failed in an earlier run is merged by the next run. @@ -414,8 +415,22 @@ def held(slug, repo, branch, accept=HELD, cache=None, token=None): """ remote = f"origin/{branch}" base = rm.git(repo, "merge-base", "origin/main", remote) - out = rm.git(repo, "rev-list", "--reverse", f"{base}..origin/main") - commits = out.split() if out else [] + # First parent, no merges: a merge-back's merge and the old release commits it brings in are not main's own. The + # bot's direct commits (RC Helm bumps) are build-generated, as relmgr.classify_commit classes them. + out = rm.git( + repo, + "log", + "--reverse", + "--first-parent", + "--no-merges", + "--format=%H %ae", + f"{base}..origin/main", + ) + commits = [ + sha + for sha, email in (row.split(" ", 1) for row in out.splitlines()) + if email.lower() != rm.BOT_EMAIL.lower() + ] classes = {} if cache is None else cache looked = 0 for s in commits: @@ -575,7 +590,9 @@ def pick_row(slug, sha, b, cls, pr, title): def rescan(slug, targets, confirm, listed): - """Push mode: (rows, awaiting, {branch: missing}) over every target since its cut (see the module doc).""" + """Push mode: (rows, awaiting, {branch: missing}) over every target since its cut (see the module doc). + Known limit: no test drives rescan itself; rescan_plan and held are tested on their own. + """ scan, others = rescan_targets(targets, listed) rows = [] for b in others: @@ -719,6 +736,9 @@ def finish_open(slug, branch, head): failed in an earlier run: squash it now, unless it is a conflict pick that lost its label. A conflict PR or a person's commits wait for a person; a head with no open PR waits PICK_TIMEOUT for the run that pushed it, then fails until a person deletes it or reopens its PR. + Known limits: the open PR's author is not checked, so a PR a person opened on a backport/* head is squashed like + the bot's when it holds only a pick. A head whose PR was closed but whose decline record failed also reads as + "no open PR": deleting that head re-picks the declined fix; re-run the record job instead. """ pending = "skipped", f"conflict PR pending on `{head}`" prs = json.loads( @@ -807,7 +827,7 @@ def pick(slug, sha, branch, title, src): "git", "ls-remote", "--exit-code", "--heads", "origin", head, check=False ).returncode == 0 - ): + ): # known limit: any other ls-remote failure (network, auth) reads as "no head", and the pick push then fails return finish_open(slug, branch, head) labels = [x for name in fix_labels(slug, src) for x in ("--label", name)] run("git", "checkout", "-q", "--detach", remote) @@ -910,6 +930,7 @@ def push_title(slug, sha): try: cls, pr, _ = fix_class(slug, sha) except ValueError as e: # a title Lint PR should have stopped: the picks are done, only this line is left + # Known limit: a gh failure (RuntimeError) is not caught here and ends the run before its summary. print(f"::warning::{sha[:7]}: fix class: {e}") cls, pr = UNCLASSIFIABLE, "-" return f"Backport · push {rm.commit_link(sha)} ({link_pr(pr)}, {cls or 'not a fix'})" diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index b4545893b3..769039f1f4 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -19,7 +19,7 @@ head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. -Run from a full clone of main, GH_TOKEN = bot token. +Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. """ import argparse @@ -155,6 +155,7 @@ def squash(slug, pr, msg, sleep=time.sleep): except RuntimeError as e: # A required check GitHub still waits for: the caller reports it. Accepted: this matches GitHub's # "Required status check ... is expected/in progress" wording, which GitHub does not document as stable. + # Known limit: a 405 naming a failed status check matches too and reads as pending. if "status check" in str(e).lower(): print(f"#{pr['number']}: {str(e).splitlines()[0]}") return False @@ -278,11 +279,17 @@ def record(slug, n, sender): raise RuntimeError( f"#{n} is not a backport PR ({head} into {base}): nothing recorded" ) - # A head name and trailer are strings any writer can push; only a PR the bot opened is its pick. GH_TOKEN is the - # bot token Backport opens pick PRs with, so its own login is the one to compare. - if (author := pr["author"]["login"]) != (bot := gh("api", "user", "--jq", ".login")): + # A head name and trailer are strings any writer can push; only a PR the bot opened is its pick. Backport opens + # pick PRs with the release App's installation token, so they are authored by app/. That token cannot + # read `gh api user`, so the slug comes from the workflow. + if not (app := os.environ.get("RELEASE_BOT_APP_SLUG")): raise RuntimeError( - f"#{n} was opened by @{author}, not by the bot (@{bot}): nothing recorded" + f"#{n}: RELEASE_BOT_APP_SLUG is unset (the release App's slug): nothing recorded" + ) + author, bot = pr["author"], f"app/{app}" + if not author.get("is_bot") or author["login"] != bot: + raise RuntimeError( + f"#{n} was opened by @{author['login']}, not by the bot (@{bot}): nothing recorded" ) if pr["state"] == "OPEN": set_label(slug, pr, CONFLICT) diff --git a/.github/release/convgate.py b/.github/release/convgate.py index 3542234985..d960e0fef3 100644 --- a/.github/release/convgate.py +++ b/.github/release/convgate.py @@ -8,7 +8,9 @@ - scope: relstate `supported[T]` minus the branch. Retired and not-enabled branches are out by construction. - direction: an older branch's picks are checked against the newer branch, whichever side is being promoted. - reference: a newer branch with a promoted stable tag for T is scanned at that tag (what ships next), not at its tip, - even before it is GA (on-prem: promoted, chart not published yet). The branch being promoted is scanned at --promote-ref. + even before it is GA (on-prem: promoted, chart not published yet). The branch being promoted is scanned at + --promote-ref where it is the newer side. As the older side it is scanned at its tip, deliberately: a fix picked + onto it after the RC still blocks the promote until the newer line has it. - declines shown: an older line's pick the newer line declined prints as DECLINED, with the reason and the decline commit (relmgr.decline_records). It counts as converged. A decline on the older line owes the newer line nothing. - release-branch-only window: an RBO-ABSENT row is kept only for a commit dated after the newer line's latest GA @@ -117,7 +119,10 @@ def missing_message( if unshipped: # nothing to pick or decline: the fix only waits for newer's next patch return f"{head}: {fix} is on {newer} but not in its latest stable {unshipped}: promote a {newer} patch first." if branch == older: - msg = f"{head}: {fix} is not on newer {newer} yet." + msg = ( + f"{head}: {fix} is not on newer {newer} yet; an older line never ships a fix its newer line lacks, " + "even one picked after the RC being promoted." + ) act = ( f"Conflict PR #{pr[0]} on {newer}: resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." if pr diff --git a/.github/release/oss.py b/.github/release/oss.py index 9e051143c9..7d55d60dce 100644 --- a/.github/release/oss.py +++ b/.github/release/oss.py @@ -188,6 +188,7 @@ def gate(oss, branch, ref, annotate): return r.returncode +# Known limit: line is not checked to be X.Y. def cut(oss, line, slack_url=""): """Cut OSS release/ from OSS main's tip plus its first RC. Refused when the OSS line exists, unless it is this cut's own result from an earlier run (its tip the bot bump carrying ee-v.0-rc.1), or when the minor @@ -242,6 +243,7 @@ def rc(oss, branch, slack_url=""): return tag +# Known limit: force accepts an empty reason. def promote(repo, oss, cloud_rc, force=False, reason=""): """The stable OSS tag of the OSS RC cloud_rc pins, tagging it when it has none yet. Open backport conflict PRs on the OSS line (or a failed listing of them) and its gate block first unless force; the other cloud product finds diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py index c178a6a0d2..cacb303edf 100644 --- a/.github/release/relmgr.py +++ b/.github/release/relmgr.py @@ -13,7 +13,8 @@ # Anywhere in the message, at a line start; `* ` too, because a squash with COMMIT_MESSAGES lists each commit's # subject as `* ` and the bot's Release-branch-only commit is subject-only. RBO_RE = re.compile(r"^(?:\* )?Release-branch-only: (\S+)", re.M) -# `UN-1 [FIX] x` or `[FIX] x`: the ticket is optional. +# `UN-1 [FIX] x` or `[FIX] x`: the ticket is optional. Known limit: one ticket only, so `UN-1 UN-2 [FIX] x` fails +# Lint PR and is unclassifiable for Backport. PR_TITLE_RE = re.compile( r"^(?:(?P[A-Z][A-Z0-9]*-\d+) )?\[(?PFIX|HOT-FIX|FEAT|GATED-FEAT|DOCS|DEPS|MISC|DEV|WIP)\] \S" ) @@ -249,6 +250,8 @@ def ci_commits( and not pick_trailer(head_ref, git(repo, "log", "-1", "--format=%B", c)) ): kind = "invalid" + # Known limit: the label passes every invalid commit; it does not check that a Release-branch-only line will + # be in the squash (a fork PR, or a merge before the bot pushes that commit). rows.append( (c, "release-branch-only (label)" if labelled and kind == "invalid" else kind) ) @@ -440,7 +443,9 @@ def promote_version(repo, rc_tag): def is_converged(repo, c_main, branch, main="main"): - """Inherited (ancestor) or picked/declined after the cut (trailer in merge-base..branch).""" + """Inherited (ancestor) or picked/declined after the cut (trailer in merge-base..branch). + Known limit: no test covers the inherited path (a commit from before the cut) on its own. + """ full = git(repo, "rev-parse", f"{c_main}^{{commit}}") if is_ancestor(repo, full, branch): return True diff --git a/.github/release/relstate.py b/.github/release/relstate.py index d2c9e63b1a..4fd3dd21cc 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -22,7 +22,7 @@ import relmgr as rm -# On-prem support window from the line's latest stable chart push, so each patch restarts it. +# On-prem support window from the line's first stable chart push (its GA); a patch does not extend it. WINDOW = dt.timedelta(days=184) # 184 days covers the longest 6-month span WINDOW_LINES = ( 6 # and only the newest 6 on-prem GA lines are supported, whichever ends first @@ -259,10 +259,12 @@ def rc_slack(repo, tag): def target_state(t, name, rows, now): """State of target t on an enabled branch. The one place the GA/supported convention lives. - On-prem: stabilized for WINDOW after the line's latest stable chart push (a patch restarts it), and only while + On-prem: stabilized for WINDOW after the line's first stable chart push (a patch does not extend it), and only while among the newest WINDOW_LINES lines with an on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilized from its GA while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. + Known limit: a KEPT target is stabilized from its first ee-vX.Y.0 while its cloud line may still be stabilizing, + so plain fixes stop going to the OSS line before the cloud line's GA. """ ga = rows[name]["targets"][t]["ga"] if not ga: @@ -296,12 +298,9 @@ def _main(repo): def _next_minor(repo, ordered): """One minor past the highest release/X.Y line, shipped retired line or stable vX.Y.Z on main, so a cut never - reuses a minor shipped by this flow. A line retired early before any GA (retired/X.Y-, its RCs deleted) is - reused. + reuses a shipped minor. A line retired early before any GA (retired/X.Y-, its RCs deleted) is reused. - With no release/X.Y branch yet (cutover) only the tags count. A stable tag that is not on main does not count - (that keeps strays such as v100.0.x out), so an old-flow release not yet merged back can be proposed again: - check the proposal against `git tag -l` before the first cut. + With no release/X.Y branch yet (cutover) only the tags count. """ main = _main(repo) lines = [ @@ -397,12 +396,13 @@ def derive(repo, now, registry=registry_tags, rc=None): # On-prem GA reads the chart artifact: RC chart pushes are tagged 0.0.0-rc.N and match no line; the # stable chart push is tagged vX.Y.Z (push-charts-to-gcr.yaml checks it against the release tag). stable = re.compile(rf"{re.escape(PREFIX)}v{re.escape(row['line'])}\.\d+") - # ga anchors state and the support window; promoted (the build artifact's latest stable) is what ships next - for key, artifact in (("ga", anchor), ("promoted", builds)): + # ga (the anchor's first stable) anchors state and the support window; promoted (the build artifact's + # latest stable) is what ships next + for key, artifact, pick in (("ga", anchor, 0), ("promoted", builds, -1)): if tags := sorted( (g for g in reg[artifact] if stable.fullmatch(g)), key=_vkey ): - tt[key] = {"tag": tags[-1], **reg[artifact][tags[-1]]} + tt[key] = {"tag": tags[pick], **reg[artifact][tags[pick]]} for name, row in rows.items(): for t, tt in row["targets"].items(): @@ -501,6 +501,7 @@ def abandon_plan(repo, branch, registry=registry_tags): versions, kept = {}, [] for a, tags in reg.items(): by_digest = {} + # Known limit: string order, so the version kept as last below can be rc.9's when rc.10 exists. for t, info in sorted(tags.items()): by_digest.setdefault( info["digest"], @@ -539,9 +540,8 @@ def promote_blockers(state, branch, target): promote retag or the on-prem chart push. A retired line is gone from the state, so it never blocks. """ # Not applied to a KEPT target (OSS). The order protects customers who stay on a line and upgrade later - # (on-prem windows). OSS community users run only :latest, which moves only on the newest line, so an older - # line shipping after a newer one reaches nobody. Each OSS line goes GA with its cloud line, so the order - # would hold a community release on the newest line until the previous cloud line is GA. + # (on-prem windows). Each OSS line goes GA with its cloud line, so the order would hold an OSS line until the + # previous cloud line is GA. if target in KEPT: return [] line = _key(branch.split("/", 1)[1]) diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index 1f5f93eb74..3d94c72b65 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -149,7 +149,26 @@ def test_classes_and_fallback(self): git("commit", "-q", "--allow-empty", "-m", f"c{i}") shas.append(git("rev-parse", "HEAD")) cut, fix, sec, misc, bad, nopr = shas - git("update-ref", "refs/remotes/origin/main", nopr) + # Not held: the bot's direct commit, and a merge-back with the old release commit it brings in. + rm.git( + repo, + "-c", + "user.name=bot", + "-c", + f"user.email={rm.BOT_EMAIL}", + "commit", + "-q", + "--allow-empty", + "-m", + "Update Helm files for RC rc.1", + ) + git("checkout", "-q", "-b", "old", cut) + git("commit", "-q", "--allow-empty", "-m", "old release fix") + git("checkout", "-q", "main") + git( + "merge", "-q", "--no-ff", "old", "-m", "Merge release v0.0.1 back to main" + ) + git("update-ref", "refs/remotes/origin/main", "HEAD") git("update-ref", "refs/remotes/origin/release/0.1", cut) pr = lambda n, sha, title, labels=(): { "number": n, @@ -373,6 +392,47 @@ def test_row_names_the_route(self): self.assertEqual(awaiting, []) +class DispatchHold(unittest.TestCase): + def test_security_waits_for_rm_on_a_stabilized_line(self): + sec, fix = "s" * 40, "f" * 40 + classes = { + sec: ("security", "#3", "UN-3 [FIX] x"), + fix: ("fix", "#4", "UN-4 [FIX] y"), + } + targets = { + "fix": ["release/0.191"], + "security_critical": [ + {"branch": "release/0.191", "needs_rm_confirmation": False}, + {"branch": "release/0.190", "needs_rm_confirmation": True}, + ], + } + with ( + mock.patch.object(bp, "resolve", lambda slug, c: c), + mock.patch.object(bp, "main_order", lambda repo, shas: shas), + mock.patch.object(bp, "fix_class", lambda slug, sha: classes[sha]), + mock.patch.object(bp, "pick_row", lambda slug, sha, b, *a: (sha, b)), + ): + run = lambda confirm: bp.dispatch("o/r", targets, f"{sec} {fix}", confirm, []) + self.assertEqual( + run(False), + ( + [(sec, "release/0.191"), (fix, "release/0.191")], + [(sec, "release/0.190", "security", "#3")], + ), + ) + self.assertEqual( + run(True), + ( + [ + (sec, "release/0.191"), + (sec, "release/0.190"), + (fix, "release/0.191"), + ], + [], + ), + ) + + class PushSummary(unittest.TestCase): def setUp(self): # plain-text links, whatever the shell exports self.env = mock.patch.dict( diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index c415fe8cdb..ab7eb3b5b1 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -14,7 +14,9 @@ O = "c" * 40 # the main original CLOSED = "2026-10-12T10:00:00Z" ME = "closer" -BOT = "release-bot" +APP = "zipstack-push-to-main" # the release App: gh pr view shows its PRs authored by app/, is_bot true +BOT = f"app/{APP}" +os.environ["RELEASE_BOT_APP_SLUG"] = APP def comment(body, at=CLOSED, user=ME): @@ -97,7 +99,7 @@ def __init__(self, state, labels=(), comments=()): "headRefOid": "a" * 40, "labels": [{"name": n} for n in labels], "isCrossRepository": False, - "author": {"login": BOT}, + "author": {"login": BOT, "is_bot": True}, } self.comments, self.calls = list(comments), [] @@ -105,8 +107,6 @@ def __call__(self, *args, token=None): self.calls.append(args) if args[:2] == ("pr", "view"): return json.dumps(self.pr) - if args[:2] == ("api", "user"): - return BOT if args[:2] == ("api", "--paginate"): return "\n".join(json.dumps(c) for c in self.comments) if args[:4] == ("api", "-X", "POST", "repos/o/r/git/commits"): @@ -162,7 +162,9 @@ def test_merged(self): ("-X", "DELETE", "repos/o/r/git/refs/heads/backport/0.187/ccccccc"), ], ) - self.assertEqual(bm.gh.calls[4][-1], "labels[]=backport-merged") + self.assertEqual( + bm.gh.calls[3][-1], "labels[]=backport-merged" + ) # after pr view and two deletes def test_close_with_comment_declines_and_keeps_head(self): out = self.close( @@ -274,7 +276,9 @@ def test_not_a_backport_pr_refused(self): {"headRefName": "backport/0.186/ccccccc"}, {"headRefName": "backport/0.187/ccccccc-x"}, {"headRefName": "backport/0.187/cccccc"}, - {"author": {"login": ME}}, + {"author": {"login": ME, "is_bot": False}}, + {"author": {"login": BOT, "is_bot": False}}, + {"author": {"login": "app/other-app", "is_bot": True}}, ): bm.gh = Gh("CLOSED", labels=["backport-conflict"]) bm.gh.pr.update(change) @@ -282,6 +286,13 @@ def test_not_a_backport_pr_refused(self): bm.close("o/r", 7, ME) self.assertEqual(bm.gh.writes(), []) + def test_unset_app_slug_refused(self): + bm.gh = Gh("CLOSED", labels=["backport-conflict"]) + with mock.patch.dict(os.environ, {"RELEASE_BOT_APP_SLUG": ""}): + with self.assertRaisesRegex(RuntimeError, "RELEASE_BOT_APP_SLUG is unset"): + bm.close("o/r", 7, ME) + self.assertEqual(bm.gh.writes(), []) + def test_head_already_gone(self): def gh(*args, token=None): if args[:3] == ("api", "-X", "DELETE") and "/git/refs/" in args[3]: diff --git a/.github/release/test_convgate.py b/.github/release/test_convgate.py index 5c3de38835..80d9594cef 100644 --- a/.github/release/test_convgate.py +++ b/.github/release/test_convgate.py @@ -292,7 +292,7 @@ def test_7a_older_promote_names_conflict_pr(self): ) self.assertEqual(code, 2) msg = ( - f"Cannot promote release/1.5 for cloud: fix {c7} is not on newer release/1.6 yet. " + f"Cannot promote release/1.5 for cloud: fix {c7} is not on newer release/1.6 yet; an older line never ships a fix its newer line lacks, even one picked after the RC being promoted. " "Conflict PR #12 on release/1.6: resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." ) self.assertIn(f"::error::{msg}", self.printed) @@ -318,7 +318,8 @@ def test_7c_no_pr_fallback_warns(self): self.assertEqual(code, 2) msg = ( f"Promote of release/1.5 for cloud will be blocked: fix {d[:7]} (PR #9) is not on newer release/1.6 " - f"yet. No Backport PR found for it: run Backport for {d[:7]}, or decline it on release/1.6." + "yet; an older line never ships a fix its newer line lacks, even one picked after the RC being promoted. " + f"No Backport PR found for it: run Backport for {d[:7]}, or decline it on release/1.6." ) self.assertIn(f"::warning::{msg}", self.printed) self.assertNotIn("::error::", self.printed) @@ -327,7 +328,7 @@ def test_7c_no_pr_fallback_warns(self): "release/1.5", failures={(9, "release/1.6"): "https://x/runs/5"} ) self.assertIn( - f"(PR #9) is not on newer release/1.6 yet. Backport for {d[:7]} to release/1.6 failed: push " + f"(PR #9) is not on newer release/1.6 yet; an older line never ships a fix its newer line lacks, even one picked after the RC being promoted. Backport for {d[:7]} to release/1.6 failed: push " f"rejected (run https://x/runs/5). Rerun Backport for {d[:7]} on release/1.6.", out, ) diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index a57c6986bb..0b478ddce5 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -219,6 +219,14 @@ def test_5_onprem_window_over(self): self.assertEqual(st["supported"], {"cloud": ["release/1.5"], "on-prem": []}) self.assertEqual(st["retire"], []) + def test_5b_onprem_window_from_first_ga(self): + # A patch chart push does not extend the window: it runs from the line's first stable chart push. + self.s4(chart_days=200) + self.push("chart/unstract-platform", "v1.5.2", days=1) + tt = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] + self.assertEqual(tt["state"], "past-eos") + self.assertEqual(tt["ga"], self.ga("chart/unstract-platform", "v1.5.1")) + def test_6_newer_cloud_ga(self): self.s6() st = self.state() @@ -924,6 +932,21 @@ def test_34_reverted_pick_only_on_its_own_branch(self): ) ) + def test_35b_forged_pick_trailer_is_invalid(self): + # A trailer naming a commit that is not on main is not a pick of a main original. + self.cut("1.5") + work = self.g("rev-parse", "release/1.5") # on the line only + before = work + forged = self.commit(f"fix\n\n(cherry picked from commit {work})", "release/1.5") + self.assertEqual(rm.classify_commit(self.repo, forged, main="main"), "invalid") + self.g("checkout", "-q", "release/1.5") + self.assertEqual( + rm.ci_commits(self.repo, "release/1.5", "push", before=before, main="main")[ + 0 + ], + [(forged, "invalid")], + ) + def test_35_ci_commits(self): fix = self.commit("fix on main") self.cut("1.5") # one invalid commit, `work on 1.5` diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 0d4fc14b03..2afc4234d7 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -133,6 +133,7 @@ jobs: - name: Record the close env: GH_TOKEN: ${{ steps.bot.outputs.token }} # labels, comments, the decline commit and the head delete + RELEASE_BOT_APP_SLUG: ${{ steps.bot.outputs.app-slug }} # the pick PR's author must be this App PR: ${{ github.event.pull_request.number }} # who closed it: their close comment is the decline reason SENDER: ${{ github.event.sender.login }} From 338572001fb2a03ff7cfde85aee3383ae193a257 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 14:34:23 +0530 Subject: [PATCH 07/37] fix: name the trailer job in the header and say a failed check failed Self-review round 2, N5: the header said "the bot" never pushes to main, release/* or backport/*, but Backport's bot pushes backport/* heads; only job `trailer` has that guard. The merge note said the commit check "did not finish" when it had failed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/release-branch-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-branch-ci.yaml b/.github/workflows/release-branch-ci.yaml index 955e4254e2..c4154ba720 100644 --- a/.github/workflows/release-branch-ci.yaml +++ b/.github/workflows/release-branch-ci.yaml @@ -13,7 +13,7 @@ name: Release branch CI # merge keeps it in the message, and the push run checks that merged commit like any other. It holds the bot token, # so it runs on pull_request_target: the base line's copy of this file, which a PR cannot edit. A line cut before # that trigger existed gets no bot commit; add the line to a commit yourself there. -# Label changes on a closed PR run nothing. The bot never pushes to a head named main, release/* or backport/*. +# Label changes on a closed PR run nothing. Job `trailer` never pushes to a head named main, release/* or backport/*. # No push builds an RC: a security or critical pick lands and waits for the RM's Take RC build, like any other pick. on: push: @@ -90,7 +90,7 @@ jobs: echo "::notice::$hint" echo "$hint" >> "$GITHUB_STEP_SUMMARY" elif [ "$OUTCOME" != success ]; then - echo "The commit check did not finish; see its log. Do not merge until it passes." >> "$GITHUB_STEP_SUMMARY" + echo "The commit check failed or did not run; see the job log. Do not merge until it passes." >> "$GITHUB_STEP_SUMMARY" elif [ "$RBO_LABEL" = true ]; then echo "Squash and merge: the \`Release-branch-only: \` commit the bot added carries the line." >> "$GITHUB_STEP_SUMMARY" else From 72d0eb22e38bad3ae4da27555ac4bac95636d70a Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 14:39:23 +0530 Subject: [PATCH 08/37] fix: take the release tooling's round-2 fixes; pass the App slug to pick .github/release is copied from the enterprise repo's tree b2f9c043: a leftover pick PR is squashed only when the release App opened it for its pick, the promoted index is tested, and several comments are corrected. The pick job now passes RELEASE_BOT_APP_SLUG, which finish_open reads for that author check. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 38 ++++++++++++++++++------------- .github/release/backport_merge.py | 33 ++++++++++++++++----------- .github/release/relmgr.py | 10 ++++---- .github/release/relstate.py | 4 +++- .github/release/test_backport.py | 31 ++++++++++++++++++++++++- .github/release/test_relstate.py | 5 ++++ .github/workflows/backport.yaml | 1 + 7 files changed, 86 insertions(+), 36 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index 695551de87..30248bc360 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -26,7 +26,7 @@ A rejected push of the pick head is retried (3 attempts, 5 s then 20 s apart); a final rejection is one sentence naming the cause and the fix, in the run's ::error::, on the source PR and in the gate's MISSING row. Conflict: committed with markers and left open with label backport-conflict. A person resolves it, gets one -approving review and merges it, or closes it; the Backport PR workflow (backport_merge.py) then records the merge, +approving review and merges it, or closes it; Backport's record job (backport_merge.py) then records the merge, or the close as a decline (the close comment is the reason). Reopening and merging undoes a decline. Never builds. Targets come from relstate's backport_targets only, never from input: a branch is a target when its state accepts the fix. --branches only narrows those targets; it never adds one. @@ -732,13 +732,13 @@ def fix_labels(slug, src): def finish_open(slug, branch, head): - """The pick head already exists. An open unlabelled PR with only the pick on it is a clean pick whose merge - failed in an earlier run: squash it now, unless it is a conflict pick that lost its label. A conflict PR or a - person's commits wait for a person; a head with no open PR waits PICK_TIMEOUT for the run that pushed it, then - fails until a person deletes it or reopens its PR. - Known limits: the open PR's author is not checked, so a PR a person opened on a backport/* head is squashed like - the bot's when it holds only a pick. A head whose PR was closed but whose decline record failed also reads as - "no open PR": deleting that head re-picks the declined fix; re-run the record job instead. + """The pick head already exists. An open unlabelled PR the bot opened, with one commit carrying the pick trailer + of the head's sha7, is a clean pick whose merge failed in an earlier run: squash it now, unless it is a conflict + pick that lost its label. A conflict PR or a person's commits wait for a person; a PR a person opened, or one + commit that is not that pick, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run that pushed + it, then fails until a person deletes it or reopens its PR. + Known limit: a head whose PR was closed but whose decline record failed also reads as "no open PR": deleting + that head re-picks the declined fix; re-run the record job instead. """ pending = "skipped", f"conflict PR pending on `{head}`" prs = json.loads( @@ -754,7 +754,7 @@ def finish_open(slug, branch, head): "--state", "open", "--json", - "number,headRefOid,labels", + "number,headRefOid,labels,author", ) ) if ( @@ -785,17 +785,22 @@ def finish_open(slug, branch, head): run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") if rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}") != "1": return pending - if rm.conflict_markers(REPO, pr["headRefOid"]) or CONFLICTS_RE.search( - rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) - ): + msg = rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) + # The bot squashes with a token that bypasses the release ruleset: only its own PR, holding only its pick. + if why := bm.bot_refusal(pr): + return "failed", f"{why}: not merged" + if not rm.pick_trailer(head, msg): + return ( + "failed", + f"#{pr['number']}: its commit is not the pick {head} names: not merged", + ) + if rm.conflict_markers(REPO, pr["headRefOid"]) or CONFLICTS_RE.search(msg): return "failed", ( f"#{pr['number']} is a conflict pick without the backport-conflict label: label it " "backport-conflict, then resolve and merge it, or close it to decline" ) try: - if not bm.squash( - slug, pr, rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) - ): + if not bm.squash(slug, pr, msg): return "failed", f"#{pr['number']} merge pending check" except RuntimeError as e: return "failed", f"#{pr['number']} merge retry: {str(e).splitlines()[0]}" @@ -927,10 +932,11 @@ def pick(slug, sha, branch, title, src): def push_title(slug, sha): """The push summary's head line: the pushed commit and its class.""" + # Known limit: a gh failure that survives fix_class's retry (RuntimeError) is not caught here and ends the run + # before its summary. try: cls, pr, _ = fix_class(slug, sha) except ValueError as e: # a title Lint PR should have stopped: the picks are done, only this line is left - # Known limit: a gh failure (RuntimeError) is not caught here and ends the run before its summary. print(f"::warning::{sha[:7]}: fix class: {e}") cls, pr = UNCLASSIFIABLE, "-" return f"Backport · push {rm.commit_link(sha)} ({link_pr(pr)}, {cls or 'not a fix'})" diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 769039f1f4..50a6ac239d 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -1,7 +1,7 @@ """The bot's side of a backport PR: Backport's squash of a clean pick, and the reaction when a person closes or reopens one. -Usage: backport_merge.py --pr N --sender LOGIN (Backport PR workflow, on `pull_request_target` closed or reopened +Usage: backport_merge.py --pr N --sender LOGIN (Backport's record job, on `pull_request_target` closed or reopened for a backport/* head into release/*; LOGIN is the event's sender; a failed run is recovered with "Re-run jobs"). Anything but a same-repo backport/X.Y/ head into release/X.Y, opened by the bot, is refused. @@ -261,6 +261,23 @@ def close(slug, n, sender): return record(slug, n, sender)["result"] +def bot_refusal(pr): + """Why `pr` (gh's number and author) is not a PR the bot opened, or None. + + A head name and trailer are strings any writer can push; only a PR the bot opened is its pick. Backport opens + pick PRs with the release App's installation token, so they are authored by app/. That token cannot + read `gh api user`, so the slug comes from the workflow. + """ + if not (app := os.environ.get("RELEASE_BOT_APP_SLUG")): + return f"#{pr['number']}: RELEASE_BOT_APP_SLUG is unset (the release App's slug)" + author, bot = pr["author"], f"app/{app}" + if not author.get("is_bot") or author["login"] != bot: + return ( + f"#{pr['number']} was opened by @{author['login']}, not by the bot (@{bot})" + ) + return None + + def record(slug, n, sender): """What close did: {result, base, sha, reason}; sha is the squash, decline or existing pick, reason the decline's.""" pr = json.loads( @@ -279,18 +296,8 @@ def record(slug, n, sender): raise RuntimeError( f"#{n} is not a backport PR ({head} into {base}): nothing recorded" ) - # A head name and trailer are strings any writer can push; only a PR the bot opened is its pick. Backport opens - # pick PRs with the release App's installation token, so they are authored by app/. That token cannot - # read `gh api user`, so the slug comes from the workflow. - if not (app := os.environ.get("RELEASE_BOT_APP_SLUG")): - raise RuntimeError( - f"#{n}: RELEASE_BOT_APP_SLUG is unset (the release App's slug): nothing recorded" - ) - author, bot = pr["author"], f"app/{app}" - if not author.get("is_bot") or author["login"] != bot: - raise RuntimeError( - f"#{n} was opened by @{author['login']}, not by the bot (@{bot}): nothing recorded" - ) + if why := bot_refusal(pr): + raise RuntimeError(f"{why}: nothing recorded") if pr["state"] == "OPEN": set_label(slug, pr, CONFLICT) return {"result": "reopened", "base": base, "sha": None, "reason": None} diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py index cacb303edf..150a0b8a22 100644 --- a/.github/release/relmgr.py +++ b/.github/release/relmgr.py @@ -13,8 +13,8 @@ # Anywhere in the message, at a line start; `* ` too, because a squash with COMMIT_MESSAGES lists each commit's # subject as `* ` and the bot's Release-branch-only commit is subject-only. RBO_RE = re.compile(r"^(?:\* )?Release-branch-only: (\S+)", re.M) -# `UN-1 [FIX] x` or `[FIX] x`: the ticket is optional. Known limit: one ticket only, so `UN-1 UN-2 [FIX] x` fails -# Lint PR and is unclassifiable for Backport. +# `UN-1 [FIX] x` or `[FIX] x`: the ticket is optional. Known limit: one ticket only, so `UN-1 UN-2 [FIX] x` without +# a fix/ label fails Lint PR and is unclassifiable for Backport. PR_TITLE_RE = re.compile( r"^(?:(?P[A-Z][A-Z0-9]*-\d+) )?\[(?PFIX|HOT-FIX|FEAT|GATED-FEAT|DOCS|DEPS|MISC|DEV|WIP)\] \S" ) @@ -27,10 +27,10 @@ ) # git revert's body line DECLINED_RE = re.compile( r"^Declined: (.+)$", re.M -) # the reason line Backport PR writes on a decline commit +) # the reason line Backport's record job writes on a decline commit BACKPORT_HEAD_RE = re.compile( r"backport/(\d+\.\d+)/([0-9a-f]{7})" -) # the head Backport PR opens: line, sha7; fullmatch +) # the head Backport opens a pick PR from: line, sha7; fullmatch def _web(): @@ -193,7 +193,7 @@ def pr_commits(repo, base, head_ref, head="HEAD"): def is_backport(head_ref): - """Is head_ref a branch Backport PR names (backport/X.Y/)? Any other name is judged commit by commit.""" + """Is head_ref a pick head Backport names (backport/X.Y/)? Any other name is judged commit by commit.""" return bool(BACKPORT_HEAD_RE.fullmatch(head_ref)) diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 4fd3dd21cc..0a6cd584fc 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -298,7 +298,9 @@ def _main(repo): def _next_minor(repo, ordered): """One minor past the highest release/X.Y line, shipped retired line or stable vX.Y.Z on main, so a cut never - reuses a shipped minor. A line retired early before any GA (retired/X.Y-, its RCs deleted) is reused. + reuses a minor this flow shipped. Only stable tags merged into main count, so one off main (an old-flow release + not merged back, or a stray such as v100.0.x) does not. A line retired early before any GA (retired/X.Y-, + its RCs deleted) is reused. With no release/X.Y branch yet (cutover) only the tags count. """ diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index 3d94c72b65..fcca58879f 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -11,6 +11,11 @@ os.environ["GITHUB_REPOSITORY"] = ( "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets ) +os.environ["RELEASE_BOT_APP_SLUG"] = "zipstack-push-to-main" # the release App +BOT = { + "login": "app/zipstack-push-to-main", + "is_bot": True, +} # gh pr view's author of the App's PRs import backport as bp import relmgr as rm @@ -692,12 +697,13 @@ def tearDown(self): os.chdir(self.cwd) self.tmp.cleanup() - def finish(self, labels=(), oid=None, prs=None): + def finish(self, labels=(), oid=None, prs=None, author=BOT): listing = [ { "number": 4, "headRefOid": oid or self.oid, "labels": [{"name": x} for x in labels], + "author": author, } ] squash = lambda slug, pr, msg: self.merged.append((pr["number"], msg)) or True @@ -785,6 +791,29 @@ def test_head_without_a_pr_fails(self): ), ) + def test_not_the_bots_pick_is_not_squashed(self): + # The squash bypasses the release ruleset: a person's PR on a pick head, or a commit that is not the pick + # of the head's sha7, must not land through it. + for author in ( + {"login": "dev", "is_bot": False}, + {"login": "app/other", "is_bot": True}, + ): + out = self.finish(author=author) + self.assertEqual(out[0], "failed") + self.assertIn( + "not by the bot (@app/zipstack-push-to-main): not merged", out[1] + ) + for msg in ("fix: x", "fix: x\n\n(cherry picked from commit " + "b" * 40 + ")"): + out = self.finish(oid=self.push(msg)) + self.assertEqual( + out, + ( + "failed", + "#4: its commit is not the pick backport/1.0/aaaaaaa names: not merged", + ), + ) + self.assertEqual(self.merged, []) + def test_unlabelled_conflict_pick_is_not_squashed(self): trailer = "(cherry picked from commit " + "a" * 40 + ")" for oid in ( diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index 0b478ddce5..0d41dc6b65 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -616,6 +616,11 @@ def test_25_promoted_from_build_artifact(self): self.ga("on-prem/backend", "v1.5.1"), ), ) + self.push( + "on-prem/backend", "v1.5.2", git_ref="release/1.5" + ) # a patch: promoted is the newer stable + onprem = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] + self.assertEqual(onprem["promoted"], self.ga("on-prem/backend", "v1.5.2")) def test_20_tip_rc_slack(self): self.s1() diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 2afc4234d7..36aeea1a58 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -97,6 +97,7 @@ jobs: - name: Pick env: GH_TOKEN: ${{ steps.bot.outputs.token }} + RELEASE_BOT_APP_SLUG: ${{ steps.bot.outputs.app-slug }} # a leftover pick PR is squashed only if this App opened it COMMITS: ${{ inputs.commits }} CONFIRM: ${{ inputs.confirm_stabilized }} BRANCHES: ${{ inputs.branches }} From 3703de4c9bcfa37f92982616d5548db4e977b611 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 14:47:54 +0530 Subject: [PATCH 09/37] docs: say main's record script must accept every release line's job env Self-review round 3, R3: pull_request_target runs each release line's copy of the record job, which runs main's backport_merge.py. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/backport.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 36aeea1a58..1039eb0cb1 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -132,6 +132,8 @@ jobs: token: ${{ steps.bot.outputs.token }} - name: Record the close + # Each release line runs its own copy of this job with main's script, so main's backport_merge.py must + # accept the env and arguments of every live line's copy. env: GH_TOKEN: ${{ steps.bot.outputs.token }} # labels, comments, the decline commit and the head delete RELEASE_BOT_APP_SLUG: ${{ steps.bot.outputs.app-slug }} # the pick PR's author must be this App From 50ce92c9b5db4e500950ec3819e1927d80ff52c4 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 14:53:04 +0530 Subject: [PATCH 10/37] fix: take the release tooling's round-3 fixes .github/release is copied from the enterprise repo's tree b00d22b7: finish_open refuses a PR the release App did not open before it counts commits, its docstring and refusal say the check covers the pick trailer, not the commit's content, and two comments are corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 24 ++++++++++++++---------- .github/release/relstate.py | 4 ++-- .github/release/test_backport.py | 18 +++++++++++++++++- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index 30248bc360..d951096f25 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -14,7 +14,8 @@ an earlier run is merged by the next run. --commits (dispatch): only these commits. -Run from a full clone whose `origin` pushes with the bot token, with GH_TOKEN set to the bot token. +Run from a full clone whose `origin` pushes with the bot token, with GH_TOKEN set to the bot token and +RELEASE_BOT_APP_SLUG set to the release App's slug. Every pick is a PR from backport/X.Y/ to release/X.Y titled with the original main PR's title (every pick is its own PR), so PR-title changelogs list it. The class comes from that PR's title and fix/ labels. Clean pick: the bot squash-merges the PR in the same run through backport_merge.squash with the pick's own @@ -734,11 +735,13 @@ def fix_labels(slug, src): def finish_open(slug, branch, head): """The pick head already exists. An open unlabelled PR the bot opened, with one commit carrying the pick trailer of the head's sha7, is a clean pick whose merge failed in an earlier run: squash it now, unless it is a conflict - pick that lost its label. A conflict PR or a person's commits wait for a person; a PR a person opened, or one - commit that is not that pick, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run that pushed - it, then fails until a person deletes it or reopens its PR. - Known limit: a head whose PR was closed but whose decline record failed also reads as "no open PR": deleting - that head re-picks the declined fix; re-run the record job instead. + pick that lost its label. A conflict PR or a person's commits on the bot's PR wait for a person; a PR a person + opened, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run + that pushed it, then fails until a person deletes it or reopens its PR. + Known limits: the trailer is checked, not the commit's content, so a writer who force-pushes the bot's open pick + PR with one commit carrying that trailer gets it squashed (a writer can already run a workflow with the App's + token). A head whose PR was closed but whose decline record failed also reads as "no open PR": deleting that + head re-picks the declined fix; re-run the record job instead. """ pending = "skipped", f"conflict PR pending on `{head}`" prs = json.loads( @@ -783,16 +786,17 @@ def finish_open(slug, branch, head): return pending pr, remote = prs[0], f"origin/{branch}" run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") + # The bot squashes with a token that bypasses the release ruleset: only its own PR, with one commit carrying + # its pick trailer (the trailer, not the content: see the docstring). + if why := bm.bot_refusal(pr): + return "failed", f"{why}: not merged" if rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}") != "1": return pending msg = rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) - # The bot squashes with a token that bypasses the release ruleset: only its own PR, holding only its pick. - if why := bm.bot_refusal(pr): - return "failed", f"{why}: not merged" if not rm.pick_trailer(head, msg): return ( "failed", - f"#{pr['number']}: its commit is not the pick {head} names: not merged", + f"#{pr['number']}: its commit lacks the pick trailer {head} names: not merged", ) if rm.conflict_markers(REPO, pr["headRefOid"]) or CONFLICTS_RE.search(msg): return "failed", ( diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 0a6cd584fc..b78f03acc3 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -298,8 +298,8 @@ def _main(repo): def _next_minor(repo, ordered): """One minor past the highest release/X.Y line, shipped retired line or stable vX.Y.Z on main, so a cut never - reuses a minor this flow shipped. Only stable tags merged into main count, so one off main (an old-flow release - not merged back, or a stray such as v100.0.x) does not. A line retired early before any GA (retired/X.Y-, + reuses a minor this flow shipped. Of the stable vX.Y.Z tags, only those merged into main count, so one off main (an + old-flow release not merged back, or a stray such as v100.0.x) does not. A line retired early before any GA (retired/X.Y-, its RCs deleted) is reused. With no release/X.Y branch yet (cutover) only the tags count. diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index fcca58879f..9497cf6e29 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -803,13 +803,29 @@ def test_not_the_bots_pick_is_not_squashed(self): self.assertIn( "not by the bot (@app/zipstack-push-to-main): not merged", out[1] ) + # A person's PR with two commits fails too, rather than waiting silently as a conflict PR. + self.g( + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "second", + ) + out = self.finish( + oid=self.g("rev-parse", "HEAD"), author={"login": "dev", "is_bot": False} + ) + self.assertEqual(out[0], "failed") for msg in ("fix: x", "fix: x\n\n(cherry picked from commit " + "b" * 40 + ")"): out = self.finish(oid=self.push(msg)) self.assertEqual( out, ( "failed", - "#4: its commit is not the pick backport/1.0/aaaaaaa names: not merged", + "#4: its commit lacks the pick trailer backport/1.0/aaaaaaa names: not merged", ), ) self.assertEqual(self.merged, []) From 208616c84e9bf05c1c6423e096280b99f9e0cde9 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:06:36 +0530 Subject: [PATCH 11/37] ci: skip Lint PR for fork PRs; run the release test suites on PRs Lint PR now runs only for a PR whose head is in this repo; a maintainer fixes a fork PR's title before merging. A new workflow runs every .github/release/test_*.py on a pull request that changes the release tooling or a workflow that runs it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/lint-pr.yaml | 2 ++ .github/workflows/release-tooling-tests.yaml | 35 ++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/release-tooling-tests.yaml diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index 6c6d4f4c6a..d274b24cda 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -9,6 +9,7 @@ name: Lint PR # Label `release-branch-only` marks a change that lives only on its release branch: refused on main. On a release # branch, Release branch CI's bot commit writes `Release-branch-only: ` (or `#` without a ticket), which # Squash and merge keeps in the message. +# A PR from a fork is not checked: the maintainer fixes its title before merging. on: pull_request: branches: [main, 'release/**'] @@ -19,6 +20,7 @@ permissions: jobs: lint-pr: + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/.github/workflows/release-tooling-tests.yaml b/.github/workflows/release-tooling-tests.yaml new file mode 100644 index 0000000000..36be3f4515 --- /dev/null +++ b/.github/workflows/release-tooling-tests.yaml @@ -0,0 +1,35 @@ +name: Release tooling tests + +# Runs every .github/release test suite when the release tooling or a workflow that runs it changes. +# The suites use only temporary local git repos and mocks: no token, no network. +on: + pull_request: + paths: + - ".github/release/**" + - ".github/workflows/backport.yaml" + - ".github/workflows/lint-pr.yaml" + - ".github/workflows/release-branch-ci.yaml" + - ".github/workflows/release-tooling-tests.yaml" + - ".github/workflows/state.yaml" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Run the suites + run: | + rc=0 + for t in .github/release/test_*.py; do + echo "::group::$t" + python3 "$t" || rc=1 + echo "::endgroup::" + done + exit "$rc" From da87df3dc527dc38fb18b53e91c0eb48bb507409 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:11:30 +0530 Subject: [PATCH 12/37] fix: take the release tooling's round-4 fixes .github/release is copied from the enterprise repo's tree cd75cc42: - an OSS line stays stabilizing until an on-prem-ga/X.Y tag exists, written by `oss.py promote --on-prem`; - finish_open refuses a PR the release App did not open before it reads the labels; - the bot's pick PR with more than one commit fails with the steps to take, instead of waiting where no summary shows it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 15 +++++++++++---- .github/release/oss.py | 32 +++++++++++++++++++++++++++++--- .github/release/relstate.py | 11 +++++++---- .github/release/test_backport.py | 25 ++++++++++++++++++------- .github/release/test_oss.py | 19 +++++++++++++++++++ .github/release/test_relstate.py | 13 ++++++++++--- 6 files changed, 94 insertions(+), 21 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index d951096f25..3b7f9d89e9 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -782,16 +782,23 @@ def finish_open(slug, branch, head): "failed", f"head {head} exists with no open PR: delete it or reopen its PR, then rerun", ) - if len(prs) != 1 or any(x["name"].startswith("backport-") for x in prs[0]["labels"]): + if len(prs) != 1: return pending pr, remote = prs[0], f"origin/{branch}" - run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") # The bot squashes with a token that bypasses the release ruleset: only its own PR, with one commit carrying - # its pick trailer (the trailer, not the content: see the docstring). + # its pick trailer (the trailer, not the content: see the docstring). A person's PR fails even when labelled: + # the record job would refuse it once merged. if why := bm.bot_refusal(pr): return "failed", f"{why}: not merged" - if rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}") != "1": + if any(x["name"].startswith("backport-") for x in pr["labels"]): return pending + run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") + # A person's commit on the bot's unlabelled PR (say a fixup after a failed squash): no conflict PR lists it. + if (n := rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}")) != "1": + return "failed", ( + f"#{pr['number']} has {n} commits, not the one pick: not merged; label it backport-conflict, then " + "merge it, or close it to decline" + ) msg = rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) if not rm.pick_trailer(head, msg): return ( diff --git a/.github/release/oss.py b/.github/release/oss.py index 7d55d60dce..0ce67e7366 100644 --- a/.github/release/oss.py +++ b/.github/release/oss.py @@ -5,7 +5,10 @@ oss.py cut --line X.Y [--slack-url URL] cut OSS release/X.Y from OSS main; prints ee-vX.Y.0-rc.1 oss.py rc --branch release/X.Y [--slack-url URL] the tag to pin for the next RC of a cloud line: the tag on its OSS line's tip, after a new OSS RC when the tip has none - oss.py promote --rc vX.Y.Z-rc.N [--force --reason R] promote the OSS RC the cloud RC pins; prints the stable tag + oss.py promote --rc vX.Y.Z-rc.N [--force --reason R] [--on-prem] + promote the OSS RC the cloud RC pins; prints the stable tag. + --on-prem (the cloud line's on-prem promote) also tags the + OSS line on-prem-ga/X.Y once, which makes it stabilized retire(): called by cloud Retire for each cloud line it retires. cut, rc and promote print one OSS tag on stdout. Every write is re-runnable: a tag already on the same commit is @@ -244,7 +247,29 @@ def rc(oss, branch, slack_url=""): # Known limit: force accepts an empty reason. -def promote(repo, oss, cloud_rc, force=False, reason=""): +def promote(repo, oss, cloud_rc, force=False, reason="", on_prem=False): + """The stable OSS tag of the OSS RC cloud_rc pins (_promote). on_prem: the cloud line's on-prem promote, so the + OSS line's on-prem-ga/X.Y tag goes on that stable tag's commit when the line has none (a later patch keeps it). + Known limit: the tag marks the promote, not the chart push after it; a failed chart push leaves the OSS line + stabilized before its cloud line's on-prem GA. + """ + stable = _promote(repo, oss, cloud_rc, force, reason) + mark = rs.ON_PREM_GA + line_of(stable).split("/", 1)[1] + if on_prem and not rs._tag_commit(oss, mark): + sha = rs._tag_commit(oss, stable) + if not sha: + raise RuntimeError(f"no OSS tag {stable}") + _tag_push( + oss, + mark, + sha, + [f"on-prem GA: {stable} (cloud {cloud_rc})"], + f"refs/tags/{mark}", + ) + return stable + + +def _promote(repo, oss, cloud_rc, force, reason): """The stable OSS tag of the OSS RC cloud_rc pins, tagging it when it has none yet. Open backport conflict PRs on the OSS line (or a failed listing of them) and its gate block first unless force; the other cloud product finds the tag and skips both. @@ -381,6 +406,7 @@ def main(argv=None): m.add_argument("--rc", required=True) m.add_argument("--force", action="store_true") m.add_argument("--reason", default="") + m.add_argument("--on-prem", action="store_true") a = p.parse_args(argv) try: if not a.oss_repo: @@ -390,7 +416,7 @@ def main(argv=None): if a.cmd == "cut" else rc(a.oss_repo, a.branch, a.slack_url) if a.cmd == "rc" - else promote(".", a.oss_repo, a.rc, a.force, a.reason) + else promote(".", a.oss_repo, a.rc, a.force, a.reason, a.on_prem) ) except RuntimeError as e: print(f"::error::{e}", file=sys.stderr) # stdout is the tag the workflow captures diff --git a/.github/release/relstate.py b/.github/release/relstate.py index b78f03acc3..ec7c402274 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -74,6 +74,8 @@ def use(slug): AR_API = "https://artifactregistry.googleapis.com/v1" +# OSS tag on-prem-ga/X.Y: the paired cloud line reached on-prem GA. OSS git cannot see the cloud registry. +ON_PREM_GA = "on-prem-ga/" def ar_repository(kind): @@ -261,16 +263,15 @@ def target_state(t, name, rows, now): On-prem: stabilized for WINDOW after the line's first stable chart push (a patch does not extend it), and only while among the newest WINDOW_LINES lines with an on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch - for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilized from its GA + for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilizing until its + paired cloud line's on-prem GA (the ON_PREM_GA tag, which `oss.py promote --on-prem` writes), then stabilized while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. - Known limit: a KEPT target is stabilized from its first ee-vX.Y.0 while its cloud line may still be stabilizing, - so plain fixes stop going to the OSS line before the cloud line's GA. """ ga = rows[name]["targets"][t]["ga"] if not ga: return "stabilizing" if t in KEPT: - return "stabilized" + return "stabilized" if rows[name]["targets"][t]["on_prem_ga"] else "stabilizing" t_ga = sorted( (n for n, r in rows.items() if r["targets"].get(t, {}).get("ga")), key=lambda n: _key(rows[n]["line"]), @@ -405,6 +406,8 @@ def derive(repo, now, registry=registry_tags, rc=None): (g for g in reg[artifact] if stable.fullmatch(g)), key=_vkey ): tt[key] = {"tag": tags[pick], **reg[artifact][tags[pick]]} + if t in KEPT: + tt["on_prem_ga"] = bool(_tag_commit(repo, ON_PREM_GA + row["line"])) for name, row in rows.items(): for t, tt in row["targets"].items(): diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index 9497cf6e29..f8049586ea 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -764,9 +764,15 @@ def test_waits_for_a_person(self): "person", ) self.g("push", "-q", "origin", "HEAD:refs/heads/backport/1.0/aaaaaaa") + # A person's commit on top of the bot's unlabelled PR: failed, so the run's summary shows it. self.assertEqual( - self.finish(oid=self.g("rev-parse", "HEAD")), pending - ) # a person's commit on top + self.finish(oid=self.g("rev-parse", "HEAD")), + ( + "failed", + "#4 has 2 commits, not the one pick: not merged; label it backport-conflict, then merge it, " + "or close it to decline", + ), + ) self.assertEqual(self.merged, []) def test_head_without_a_pr_fails(self): @@ -798,11 +804,15 @@ def test_not_the_bots_pick_is_not_squashed(self): {"login": "dev", "is_bot": False}, {"login": "app/other", "is_bot": True}, ): - out = self.finish(author=author) - self.assertEqual(out[0], "failed") - self.assertIn( - "not by the bot (@app/zipstack-push-to-main): not merged", out[1] - ) + for labels in ( + [], + ["backport-conflict"], + ): # labelled too: it never waits as a conflict PR + out = self.finish(author=author, labels=labels) + self.assertEqual(out[0], "failed") + self.assertIn( + "not by the bot (@app/zipstack-push-to-main): not merged", out[1] + ) # A person's PR with two commits fails too, rather than waiting silently as a conflict PR. self.g( "-c", @@ -819,6 +829,7 @@ def test_not_the_bots_pick_is_not_squashed(self): oid=self.g("rev-parse", "HEAD"), author={"login": "dev", "is_bot": False} ) self.assertEqual(out[0], "failed") + self.assertIn("not by the bot", out[1]) for msg in ("fix: x", "fix: x\n\n(cherry picked from commit " + "b" * 40 + ")"): out = self.finish(oid=self.push(msg)) self.assertEqual( diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 0a20d060c9..324f056207 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -218,6 +218,25 @@ def test_promote(self): with self.assertRaisesRegex(RuntimeError, "did not run"): oss.promote(self.cloud, self.oss, "v1.7.1-rc.2", force=True, reason="x") + def test_on_prem_promote_tags_on_prem_ga_once(self): + # The OSS line stays stabilizing until its cloud line's on-prem GA: only --on-prem writes the tag, once. + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") # the cloud product + self.assertEqual(self.remote("tag", "-l", "on-prem-ga/*"), "") + self.clone() + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1", on_prem=True) + ga = self.remote("rev-parse", "ee-v1.7.0^{commit}") + self.assertEqual(self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), ga) + self.work("release/1.7", "oss fix") + self.quiet(oss.rc, self.oss, "release/1.7") + self.pin("ee-v1.7.1-rc.1", "v1.7.1-rc.1") + self.clone() + self.assertEqual( + oss.promote(self.cloud, self.oss, "v1.7.1-rc.1", on_prem=True), "ee-v1.7.1" + ) + self.assertEqual(self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), ga) + def test_promote_refuses_a_downgrade(self): self.quiet(oss.cut, self.oss, "1.7") self.work("release/1.7", "oss fix") diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index 0d41dc6b65..eefbe7f17e 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -1,5 +1,5 @@ -"""Synthetic-repo tests for relstate (derive 1-11, on-prem newest six 7b, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ticketless titles 18c, Release-branch-only key 18d, release-branch-only label 18e, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, no order block on a GA line 23b, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37, Artifact Registry reads 38-41 and 39b). Run: python test_relstate.py. No workflow runs the .github/release suites yet: run all six locally before -changing this directory. +"""Synthetic-repo tests for relstate (derive 1-11, on-prem newest six 7b, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ticketless titles 18c, Release-branch-only key 18d, release-branch-only label 18e, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, no order block on a GA line 23b, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37, Artifact Registry reads 38-41 and 39b). Run: python test_relstate.py. release-tooling-tests.yaml runs all six on a pull request +that changes this directory. """ import contextlib @@ -721,7 +721,7 @@ def test_17_abandon_plan_refusals(self): def test_26_oss_repo_config(self): # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), - # supported while its branch exists. Plain v* tags are community releases from main and never count. + # supported while its branch exists once its on-prem-ga/X.Y tag is there. Plain v* tags are community releases from main and never count. self.addCleanup(rs.use, rs.SLUG) rs.use("unstract") self.state = lambda: rs.derive( @@ -735,6 +735,13 @@ def test_26_oss_repo_config(self): self.g("tag", f"v{line}.7", f"release/{line}") # never the line's GA self.cut("1.7") self.g("tag", "ee-v1.7.0-rc.1", "release/1.7") + # Stabilizing past its own ee-vX.Y.0 until the paired cloud line's on-prem GA tags it. + self.g("tag", "on-prem-ga/1.5", "release/1.5") + self.assertEqual( + [(b["branch"], b["state"]) for b in self.state()["branches"]][1:], + [("release/1.6", "stabilizing"), ("release/1.5", "stabilized")], + ) + self.g("tag", "on-prem-ga/1.6", "release/1.6") st = self.state() self.assertEqual( [(b["branch"], b["state"]) for b in st["branches"]], From 3a552c4839ed1744f48e65274f308e0b59ca1633 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:23:33 +0530 Subject: [PATCH 13/37] docs: say Lint PR's fork skip also drops the label rules, and the recovery Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/lint-pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index d274b24cda..15ff10661c 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -9,7 +9,8 @@ name: Lint PR # Label `release-branch-only` marks a change that lives only on its release branch: refused on main. On a release # branch, Release branch CI's bot commit writes `Release-branch-only: ` (or `#` without a ticket), which # Squash and merge keeps in the message. -# A PR from a fork is not checked: the maintainer fixes its title before merging. +# A PR from a fork is not checked, title or labels: the maintainer checks both before merging. A fork PR merged with a +# bad title shows as unclassifiable on each Backport run until its title or fix/ label is fixed. on: pull_request: branches: [main, 'release/**'] From ea3b9949beeaf247bba4839019d0df8047cea61a Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:26:25 +0530 Subject: [PATCH 14/37] fix: take the release tooling's round-5 fixes .github/release tree 7ea98f29, matching the enterprise PR. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 15 ++++++++++----- .github/release/test_backport.py | 8 ++++++++ .github/release/test_oss.py | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index 3b7f9d89e9..654351dd61 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -735,8 +735,8 @@ def fix_labels(slug, src): def finish_open(slug, branch, head): """The pick head already exists. An open unlabelled PR the bot opened, with one commit carrying the pick trailer of the head's sha7, is a clean pick whose merge failed in an earlier run: squash it now, unless it is a conflict - pick that lost its label. A conflict PR or a person's commits on the bot's PR wait for a person; a PR a person - opened, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run + pick that lost its label. A labelled conflict PR waits for a person; a PR a person opened, more than one commit + or none, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run that pushed it, then fails until a person deletes it or reopens its PR. Known limits: the trailer is checked, not the commit's content, so a writer who force-pushes the bot's open pick PR with one commit carrying that trailer gets it squashed (a writer can already run a workflow with the App's @@ -795,9 +795,14 @@ def finish_open(slug, branch, head): run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") # A person's commit on the bot's unlabelled PR (say a fixup after a failed squash): no conflict PR lists it. if (n := rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}")) != "1": - return "failed", ( - f"#{pr['number']} has {n} commits, not the one pick: not merged; label it backport-conflict, then " - "merge it, or close it to decline" + return ( + "failed", + f"#{pr['number']} has {n} commits, not the one pick: not merged; " + + ( + "close it, it adds nothing" + if n == "0" + else "label it backport-conflict, then merge it, or close it to decline" + ), ) msg = rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) if not rm.pick_trailer(head, msg): diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index f8049586ea..df42e8dc90 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -773,6 +773,14 @@ def test_waits_for_a_person(self): "or close it to decline", ), ) + # A head at the release tip adds nothing: close, not merge. + self.assertEqual( + self.finish(oid=self.g("rev-parse", f"{self.oid}^")), + ( + "failed", + "#4 has 0 commits, not the one pick: not merged; close it, it adds nothing", + ), + ) self.assertEqual(self.merged, []) def test_head_without_a_pr_fails(self): diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 324f056207..5c232ab57a 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -237,6 +237,22 @@ def test_on_prem_promote_tags_on_prem_ga_once(self): ) self.assertEqual(self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), ga) + def test_cli_on_prem_flag(self): + self.quiet(oss.cut, self.oss, "1.7") + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(self.cloud) # main promotes the cloud clone in the working directory + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = oss.main( + ["--oss-repo", self.oss, "promote", "--rc", "v1.7.0-rc.1", "--on-prem"] + ) + self.assertEqual((code, out.getvalue()), (0, "ee-v1.7.0\n")) + self.assertEqual( + self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), + self.remote("rev-parse", "ee-v1.7.0^{commit}"), + ) + def test_promote_refuses_a_downgrade(self): self.quiet(oss.cut, self.oss, "1.7") self.work("release/1.7", "oss fix") From 3aedb82b45ecd11f6d6be90ef317f87de0f76851 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:34:22 +0530 Subject: [PATCH 15/37] docs: drop Lint PR's claim that Backport flags a bad fork title Backport warns only on push-to-main rescans of PRs merged into main, so a fork PR merged into a release branch is never flagged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/lint-pr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index 15ff10661c..78f10644eb 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -9,8 +9,7 @@ name: Lint PR # Label `release-branch-only` marks a change that lives only on its release branch: refused on main. On a release # branch, Release branch CI's bot commit writes `Release-branch-only: ` (or `#` without a ticket), which # Squash and merge keeps in the message. -# A PR from a fork is not checked, title or labels: the maintainer checks both before merging. A fork PR merged with a -# bad title shows as unclassifiable on each Backport run until its title or fix/ label is fixed. +# A PR from a fork is not checked, title or labels: the maintainer checks both before merging. on: pull_request: branches: [main, 'release/**'] From e8f6fc94089a0012b774cfa8b09cac169d8d2945 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:35:35 +0530 Subject: [PATCH 16/37] fix: take the release tooling's round-6 fixes .github/release tree 07e99c72, matching the enterprise PR. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 5 +++-- .github/release/backport_merge.py | 17 +++++++++++++++-- .github/release/test_backport.py | 3 ++- .github/release/test_backport_merge.py | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index 654351dd61..24c74a5c06 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -735,7 +735,7 @@ def fix_labels(slug, src): def finish_open(slug, branch, head): """The pick head already exists. An open unlabelled PR the bot opened, with one commit carrying the pick trailer of the head's sha7, is a clean pick whose merge failed in an earlier run: squash it now, unless it is a conflict - pick that lost its label. A labelled conflict PR waits for a person; a PR a person opened, more than one commit + pick that lost its label. A PR carrying any backport-* label waits for a person; a PR a person opened, more than one commit or none, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run that pushed it, then fails until a person deletes it or reopens its PR. Known limits: the trailer is checked, not the commit's content, so a writer who force-pushes the bot's open pick @@ -799,7 +799,8 @@ def finish_open(slug, branch, head): "failed", f"#{pr['number']} has {n} commits, not the one pick: not merged; " + ( - "close it, it adds nothing" + f"its head has no pick on {branch}: delete {head} (this closes #{pr['number']}) and the next " + "run re-picks the fix" if n == "0" else "label it backport-conflict, then merge it, or close it to decline" ), diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 50a6ac239d..31434b725d 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -16,7 +16,8 @@ The head is kept so the PR can be reopened. A pick or decline already on the base (newest first) -> no commit, label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the - head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. + head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. A head with no commit past the + base (moved onto it) -> nothing declined, no label or comment. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. @@ -90,9 +91,12 @@ def head_messages(pr): def first_pick(pr): """(message, [original]) for the decline: the main commit the head's first commit picks, its message plus the - pick trailer. ("", []) when that commit has no single pick trailer naming the head's sha7 on main. + pick trailer. ("", []) when that commit has no single pick trailer naming the head's sha7 on main; (None, []) when + the head has no commit past the base (moved onto it), so there is nothing to decline. """ msgs = head_messages(pr) + if not msgs: + return None, [] o = rm.pick_trailer(pr["headRefName"], msgs[0] if msgs else "") if not o or not rm.on_main(".", o, "origin/main"): return "", [] @@ -209,6 +213,8 @@ def decline(slug, pr, reason): """(sha, written): the pick or decline for this PR's originals on the base, newest first, else a new decline.""" n, base = pr["number"], pr["baseRefName"] msg, originals = first_pick(pr) + if msg is None: + return None, False if not originals: gh( "pr", @@ -316,6 +322,8 @@ def record(slug, n, sender): or f"Closed by @{sender} without a reason (PR #{n})" ) sha, written = decline(slug, pr, reason) + if not sha: # no pick on the head: nothing declined; Backport re-picks the fix once the head is deleted + return {"result": "skipped: no pick", "base": base, "sha": None, "reason": None} set_label(slug, pr, DECLINED) if not any( c["body"].startswith("Recorded as a decline") @@ -358,6 +366,11 @@ def summary(n, sender, rec): head + f"**reopened** (@{sender})", "Next: merge it to land the pick, or close it again to keep the decline.", ] + if not sha: + return [ + head + + "**skipped**: its head has no commit past the base, so nothing was declined" + ] return [head + f"**skipped**: already on `{base}` as {rm.commit_link(sha)}"] diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index df42e8dc90..87677772f1 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -778,7 +778,8 @@ def test_waits_for_a_person(self): self.finish(oid=self.g("rev-parse", f"{self.oid}^")), ( "failed", - "#4 has 0 commits, not the one pick: not merged; close it, it adds nothing", + "#4 has 0 commits, not the one pick: not merged; its head has no pick on release/1.0: delete " + "backport/1.0/aaaaaaa (this closes #4) and the next run re-picks the fix", ), ) self.assertEqual(self.merged, []) diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index ab7eb3b5b1..c582df4ab8 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -259,6 +259,14 @@ def test_lost_trailer_fails_loudly(self): self.assertEqual(bm.gh.writes(), [("pr", "comment")]) self.assertTrue(bm.gh.said()[0].startswith("Bot could not record the decline")) + def test_head_without_a_pick_declines_nothing(self): + # A head moved onto the base still owes its fix: no decline, label or comment, and the run passes. + bm.first_pick = lambda pr: (None, []) + self.assertEqual( + self.close("CLOSED", labels=["backport-conflict"]), "skipped: no pick" + ) + self.assertEqual(bm.gh.writes(), []) + def test_reopened_sets_conflict_only(self): self.assertEqual(self.close("OPEN", labels=["backport-declined"]), "reopened") self.assertEqual( @@ -352,6 +360,8 @@ def test_message_from_main(self): self.assertEqual( bm.first_pick({"headRefName": "backport/0.187/ddddddd"}), ("", []) ) # not on main + with mock.patch.object(bm, "head_messages", lambda pr: []): + self.assertEqual(bm.first_pick(pr), (None, [])) # no commit past the base class Summary(unittest.TestCase): @@ -385,6 +395,13 @@ def test_blocks(self): ], ) self.assertEqual(len(bm.summary(246, ME, rec("reopened", sha=None))), 2) + self.assertEqual( + bm.summary(246, ME, rec("skipped: no pick", sha=None)), + [ + "**Backport PR** #246 on `release/0.189`: **skipped**: its head has no commit past the base, " + "so nothing was declined" + ], + ) class SquashRetry(unittest.TestCase): From 39646d31b23fb6c62fe151bfff05c631ca6db3fc Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:45:25 +0530 Subject: [PATCH 17/37] fix: record a close with no pick, and say so in Backport's header .github/release tree 811222fc, matching the enterprise PR: a closed backport PR whose head has no commit past the base gets its head deleted and one comment on how to decline by hand. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 8 +++---- .github/release/backport_merge.py | 33 ++++++++++++++++++++++---- .github/release/test_backport.py | 2 +- .github/release/test_backport_merge.py | 17 ++++++++++--- .github/workflows/backport.yaml | 1 + 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index 24c74a5c06..ff82a3cacd 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -735,8 +735,8 @@ def fix_labels(slug, src): def finish_open(slug, branch, head): """The pick head already exists. An open unlabelled PR the bot opened, with one commit carrying the pick trailer of the head's sha7, is a clean pick whose merge failed in an earlier run: squash it now, unless it is a conflict - pick that lost its label. A PR carrying any backport-* label waits for a person; a PR a person opened, more than one commit - or none, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run + pick that lost its label. A PR a person opened fails unmerged; a bot PR carrying any backport-* label waits for a + person; more than one commit or none, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run that pushed it, then fails until a person deletes it or reopens its PR. Known limits: the trailer is checked, not the commit's content, so a writer who force-pushes the bot's open pick PR with one commit carrying that trailer gets it squashed (a writer can already run a workflow with the App's @@ -799,8 +799,8 @@ def finish_open(slug, branch, head): "failed", f"#{pr['number']} has {n} commits, not the one pick: not merged; " + ( - f"its head has no pick on {branch}: delete {head} (this closes #{pr['number']}) and the next " - "run re-picks the fix" + f"its head has no pick on {branch}: delete {head} (this closes #{pr['number']}), and the next " + "Backport run treats the fix as not picked" if n == "0" else "label it backport-conflict, then merge it, or close it to decline" ), diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 31434b725d..6105d1e9d7 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -7,7 +7,7 @@ People land a conflict PR with the normal buttons (resolve, one approving review, Squash and merge) or close it. The PR's state is read again here, so a re-run or a repeated event acts on what the PR is now. Labels on backport PRs are -the bot's: each case sets exactly one `backport-*` label and removes the others. +the bot's: each case but a head with no pick sets exactly one `backport-*` label and removes the others. - merged: label `backport-merged`; delete the head if it is still there. - closed unmerged: a decline. The reason is the closer's newest comment made in the 2 minutes up to the close (the close comment), first line, a leading `Declined:` dropped; with none, `Closed by @ without a reason (PR @@ -17,7 +17,8 @@ label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. A head with no commit past the - base (moved onto it) -> nothing declined, no label or comment. + base (moved onto it) -> nothing declined and no label: the bot deletes the head, so the next Backport run treats + the fix as not picked, and comments once how to decline it by hand. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. @@ -35,6 +36,7 @@ MERGED, DECLINED, CONFLICT = "backport-merged", "backport-declined", "backport-conflict" REASON_WINDOW = datetime.timedelta(minutes=2) +NO_PICK = "Nothing to record: this PR's head has no commit past" def is_backport(head, base): @@ -210,7 +212,9 @@ def comments(slug, n): def decline(slug, pr, reason): - """(sha, written): the pick or decline for this PR's originals on the base, newest first, else a new decline.""" + """(sha, written): the pick or decline for this PR's originals on the base, newest first, else a new decline; + (None, False) when the head has no commit past the base. + """ n, base = pr["number"], pr["baseRefName"] msg, originals = first_pick(pr) if msg is None: @@ -322,7 +326,24 @@ def record(slug, n, sender): or f"Closed by @{sender} without a reason (PR #{n})" ) sha, written = decline(slug, pr, reason) - if not sha: # no pick on the head: nothing declined; Backport re-picks the fix once the head is deleted + if not sha: # no pick on the head: nothing to decline, and the kept head would fail every Backport run + delete_head(slug, head) + if not any( + c["body"].startswith(NO_PICK) and c["created_at"] >= pr["closedAt"] + for c in cs + ): + gh( + "pr", + "comment", + str(n), + "-R", + slug, + "--body", + f"{NO_PICK} {base}, so nothing was declined. Deleted `{head}`: the next Backport run treats the fix " + f"as not picked. To decline it, push an empty commit onto {base} with the pick's message, its " + "`(cherry picked from commit

)` line and a final `Declined: ` line, or ask the " + "release manager.", + ) return {"result": "skipped: no pick", "base": base, "sha": None, "reason": None} set_label(slug, pr, DECLINED) if not any( @@ -369,7 +390,9 @@ def summary(n, sender, rec): if not sha: return [ head - + "**skipped**: its head has no commit past the base, so nothing was declined" + + "**skipped**: its head has no commit past the base, so nothing was declined", + "Next: the head is deleted and the next Backport run treats the fix as not picked; the PR comment says " + "how to decline it by hand.", ] return [head + f"**skipped**: already on `{base}` as {rm.commit_link(sha)}"] diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index 87677772f1..8e6ccf9e93 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -779,7 +779,7 @@ def test_waits_for_a_person(self): ( "failed", "#4 has 0 commits, not the one pick: not merged; its head has no pick on release/1.0: delete " - "backport/1.0/aaaaaaa (this closes #4) and the next run re-picks the fix", + "backport/1.0/aaaaaaa (this closes #4), and the next Backport run treats the fix as not picked", ), ) self.assertEqual(self.merged, []) diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index c582df4ab8..c17a6972e1 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -260,12 +260,21 @@ def test_lost_trailer_fails_loudly(self): self.assertTrue(bm.gh.said()[0].startswith("Bot could not record the decline")) def test_head_without_a_pick_declines_nothing(self): - # A head moved onto the base still owes its fix: no decline, label or comment, and the run passes. + # A head moved onto the base still owes its fix: no decline or label; the bot deletes the head, which would + # fail every later Backport run, and says once how to decline by hand. The run passes. bm.first_pick = lambda pr: (None, []) + delete = ("-X", "DELETE", "repos/o/r/git/refs/heads/backport/0.187/ccccccc") self.assertEqual( self.close("CLOSED", labels=["backport-conflict"]), "skipped: no pick" ) - self.assertEqual(bm.gh.writes(), []) + self.assertEqual(bm.gh.writes(), [delete, ("pr", "comment")]) + said = bm.gh.said()[0] + self.assertTrue( + said.startswith(bm.NO_PICK + " release/0.187, so nothing was declined") + ) + self.assertIn("Declined: ", said) + out = self.close("CLOSED", comments=[comment(said, user="bot")]) # a re-run + self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [delete])) def test_reopened_sets_conflict_only(self): self.assertEqual(self.close("OPEN", labels=["backport-declined"]), "reopened") @@ -399,7 +408,9 @@ def test_blocks(self): bm.summary(246, ME, rec("skipped: no pick", sha=None)), [ "**Backport PR** #246 on `release/0.189`: **skipped**: its head has no commit past the base, " - "so nothing was declined" + "so nothing was declined", + "Next: the head is deleted and the next Backport run treats the fix as not picked; the PR " + "comment says how to decline it by hand.", ], ) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 1039eb0cb1..a44a07047f 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -16,6 +16,7 @@ run-name: >- # or close it. The bot records what happened (backport_merge.py --pr N): # merged -> label backport-merged, head deleted; closed unmerged -> a decline: empty decline commit on the base # (the closer's close comment is the reason), label backport-declined, head kept so the PR can be reopened; +# a head with no commit past the base declines nothing: the bot deletes it and comments how to decline by hand; # reopened -> label backport-conflict. Reopen and merge undoes a decline: the newest trailer on the line wins. # pull_request_target, not pull_request: GitHub runs no pull_request workflow on a PR that conflicts with its base # at the git level (no merge ref), and a conflict PR whose base later changed the same lines is one. From 360767f7bdacf350a86d30bcbc2678903a4933d7 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 15:55:43 +0530 Subject: [PATCH 18/37] fix: delete a no-pick head only while it is still the PR's commit Port .github/release tree b3070439 from the enterprise copy: the no-pick close checks the head ref's sha before deleting, so a re-run cannot close a later re-pick PR on the same name. The comment no longer claims a delete or a direct push to the release branch, and the dedupe's close-time bound is tested. The Backport header says the delete is conditional. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport_merge.py | 33 +++++++++++++++++++------- .github/release/test_backport_merge.py | 22 +++++++++++++---- .github/workflows/backport.yaml | 3 ++- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 6105d1e9d7..365f1cd51a 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -17,8 +17,8 @@ label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. A head with no commit past the - base (moved onto it) -> nothing declined and no label: the bot deletes the head, so the next Backport run treats - the fix as not picked, and comments once how to decline it by hand. + base (moved onto it) -> nothing declined and no label: the bot deletes the head while it is still the PR's commit, + so the next Backport run treats the fix as not picked, and comments once how to decline it by hand. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. @@ -200,6 +200,18 @@ def delete_head(slug, head): raise +def head_sha(slug, head): + """The head branch's commit, or None when it is gone.""" + try: + return json.loads(gh("api", f"repos/{slug}/git/ref/heads/{head}"))["object"][ + "sha" + ] + except RuntimeError as e: + if "404" not in str(e) and "Not Found" not in str(e): + raise + return None + + def comments(slug, n): out = gh( "api", @@ -327,7 +339,10 @@ def record(slug, n, sender): ) sha, written = decline(slug, pr, reason) if not sha: # no pick on the head: nothing to decline, and the kept head would fail every Backport run - delete_head(slug, head) + # Only while it is still this PR's commit: a later run may have re-picked onto the same name, and deleting + # that head would close its PR as a decline nobody made. + if head_sha(slug, head) == pr["headRefOid"]: + delete_head(slug, head) if not any( c["body"].startswith(NO_PICK) and c["created_at"] >= pr["closedAt"] for c in cs @@ -339,10 +354,10 @@ def record(slug, n, sender): "-R", slug, "--body", - f"{NO_PICK} {base}, so nothing was declined. Deleted `{head}`: the next Backport run treats the fix " - f"as not picked. To decline it, push an empty commit onto {base} with the pick's message, its " - "`(cherry picked from commit
)` line and a final `Declined: ` line, or ask the " - "release manager.", + f"{NO_PICK} {base}, so nothing was declined, and the bot removes `{head}` while it is still at this " + "PR's commit. The next Backport run treats the fix as not picked. To decline it, ask the release " + "manager for a hand-made empty commit with the pick's message, its `(cherry picked from commit " + "
)` line and a final `Declined: ` line.", ) return {"result": "skipped: no pick", "base": base, "sha": None, "reason": None} set_label(slug, pr, DECLINED) @@ -391,8 +406,8 @@ def summary(n, sender, rec): return [ head + "**skipped**: its head has no commit past the base, so nothing was declined", - "Next: the head is deleted and the next Backport run treats the fix as not picked; the PR comment says " - "how to decline it by hand.", + "Next: the next Backport run treats the fix as not picked; the PR comment says how to decline it by " + "hand.", ] return [head + f"**skipped**: already on `{base}` as {rm.commit_link(sha)}"] diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index c17a6972e1..24c20e591f 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -89,7 +89,7 @@ def test_landed(self): class Gh: """Stand-in for bm.gh: records each call; serves the PR view, its comments and the decline commit's sha.""" - def __init__(self, state, labels=(), comments=()): + def __init__(self, state, labels=(), comments=(), ref="a" * 40): self.pr = { "number": 7, "state": state, @@ -101,7 +101,7 @@ def __init__(self, state, labels=(), comments=()): "isCrossRepository": False, "author": {"login": BOT, "is_bot": True}, } - self.comments, self.calls = list(comments), [] + self.comments, self.calls, self.ref = list(comments), [], ref def __call__(self, *args, token=None): self.calls.append(args) @@ -109,6 +109,10 @@ def __call__(self, *args, token=None): return json.dumps(self.pr) if args[:2] == ("api", "--paginate"): return "\n".join(json.dumps(c) for c in self.comments) + if args[:2] == ("api", "repos/o/r/git/ref/heads/backport/0.187/ccccccc"): + if not self.ref: + raise RuntimeError("gh api (exit 1): Not Found (HTTP 404)") + return json.dumps({"object": {"sha": self.ref}}) if args[:4] == ("api", "-X", "POST", "repos/o/r/git/commits"): return json.dumps({"sha": "e" * 40}) return "" @@ -119,6 +123,7 @@ def writes(self): a[:2] if a[0] == "pr" else a[1:4] for a in self.calls if a[:2] not in (("pr", "view"), ("api", "--paginate"), ("api", "user")) + and not a[1].startswith("repos/o/r/git/ref/") ] def said(self): @@ -275,6 +280,15 @@ def test_head_without_a_pick_declines_nothing(self): self.assertIn("Declined: ", said) out = self.close("CLOSED", comments=[comment(said, user="bot")]) # a re-run self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [delete])) + # A re-run after the next Backport run re-picked onto the same name, or after a person deleted the head: + # that head is not this PR's, so it stays. + for ref in ("b" * 40, None): + out = self.close("CLOSED", comments=[comment(said, user="bot")], ref=ref) + self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [])) + # A comment from an earlier close does not count for this one. + early = comment(said, at="2026-10-12T09:00:00Z", user="bot") + self.close("CLOSED", comments=[early]) + self.assertEqual(bm.gh.writes(), [delete, ("pr", "comment")]) def test_reopened_sets_conflict_only(self): self.assertEqual(self.close("OPEN", labels=["backport-declined"]), "reopened") @@ -409,8 +423,8 @@ def test_blocks(self): [ "**Backport PR** #246 on `release/0.189`: **skipped**: its head has no commit past the base, " "so nothing was declined", - "Next: the head is deleted and the next Backport run treats the fix as not picked; the PR " - "comment says how to decline it by hand.", + "Next: the next Backport run treats the fix as not picked; the PR comment says how to decline " + "it by hand.", ], ) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index a44a07047f..c949c099cd 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -16,7 +16,8 @@ run-name: >- # or close it. The bot records what happened (backport_merge.py --pr N): # merged -> label backport-merged, head deleted; closed unmerged -> a decline: empty decline commit on the base # (the closer's close comment is the reason), label backport-declined, head kept so the PR can be reopened; -# a head with no commit past the base declines nothing: the bot deletes it and comments how to decline by hand; +# a head with no commit past the base declines nothing: the bot deletes it while it is still the PR's commit and +# comments how to decline by hand; # reopened -> label backport-conflict. Reopen and merge undoes a decline: the newest trailer on the line wins. # pull_request_target, not pull_request: GitHub runs no pull_request workflow on a PR that conflicts with its base # at the git level (no merge ref), and a conflict PR whose base later changed the same lines is one. From 06442fc37d5fe39022259e6203ee0bc2fe962178 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 16:05:50 +0530 Subject: [PATCH 19/37] fix: leave a no-pick head for a person to delete Port the release tooling at tree c38bc5f5: on a close with no pick on the head, record no longer deletes the head. It comments once that a person deletes the head or re-picks onto it, and how to decline the fix by hand. head_sha and its tests are removed. The backport.yaml header says the same. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport_merge.py | 35 ++++++++------------------ .github/release/test_backport_merge.py | 30 ++++++++-------------- .github/workflows/backport.yaml | 4 +-- 3 files changed, 22 insertions(+), 47 deletions(-) diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 365f1cd51a..fe2abeca38 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -17,8 +17,8 @@ label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. A head with no commit past the - base (moved onto it) -> nothing declined and no label: the bot deletes the head while it is still the PR's commit, - so the next Backport run treats the fix as not picked, and comments once how to decline it by hand. + base (moved onto it) -> nothing declined, no label and no delete: the bot comments once that the head is to be + deleted by hand or re-picked onto, and how to decline the fix by hand. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. @@ -200,18 +200,6 @@ def delete_head(slug, head): raise -def head_sha(slug, head): - """The head branch's commit, or None when it is gone.""" - try: - return json.loads(gh("api", f"repos/{slug}/git/ref/heads/{head}"))["object"][ - "sha" - ] - except RuntimeError as e: - if "404" not in str(e) and "Not Found" not in str(e): - raise - return None - - def comments(slug, n): out = gh( "api", @@ -338,11 +326,9 @@ def record(slug, n, sender): or f"Closed by @{sender} without a reason (PR #{n})" ) sha, written = decline(slug, pr, reason) - if not sha: # no pick on the head: nothing to decline, and the kept head would fail every Backport run - # Only while it is still this PR's commit: a later run may have re-picked onto the same name, and deleting - # that head would close its PR as a decline nobody made. - if head_sha(slug, head) == pr["headRefOid"]: - delete_head(slug, head) + # No pick on the head: nothing to decline. Removing the head is manual: until a person deletes it (or re-picks + # onto it), every Backport run fails on it. + if not sha: if not any( c["body"].startswith(NO_PICK) and c["created_at"] >= pr["closedAt"] for c in cs @@ -354,10 +340,10 @@ def record(slug, n, sender): "-R", slug, "--body", - f"{NO_PICK} {base}, so nothing was declined, and the bot removes `{head}` while it is still at this " - "PR's commit. The next Backport run treats the fix as not picked. To decline it, ask the release " - "manager for a hand-made empty commit with the pick's message, its `(cherry picked from commit " - "
)` line and a final `Declined: ` line.", + f"{NO_PICK} {base}, so nothing was declined. Delete `{head}` by hand, or re-pick the fix onto it; " + "until then Backport fails on it. To decline the fix, ask the release manager for a hand-made empty " + "commit with the pick's message, its `(cherry picked from commit
)` line and a final " + "`Declined: ` line.", ) return {"result": "skipped: no pick", "base": base, "sha": None, "reason": None} set_label(slug, pr, DECLINED) @@ -406,8 +392,7 @@ def summary(n, sender, rec): return [ head + "**skipped**: its head has no commit past the base, so nothing was declined", - "Next: the next Backport run treats the fix as not picked; the PR comment says how to decline it by " - "hand.", + "Next: delete the head by hand, or re-pick onto it; the PR comment says how to decline the fix by hand.", ] return [head + f"**skipped**: already on `{base}` as {rm.commit_link(sha)}"] diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index 24c20e591f..4a7c3fcafd 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -89,7 +89,7 @@ def test_landed(self): class Gh: """Stand-in for bm.gh: records each call; serves the PR view, its comments and the decline commit's sha.""" - def __init__(self, state, labels=(), comments=(), ref="a" * 40): + def __init__(self, state, labels=(), comments=()): self.pr = { "number": 7, "state": state, @@ -101,7 +101,7 @@ def __init__(self, state, labels=(), comments=(), ref="a" * 40): "isCrossRepository": False, "author": {"login": BOT, "is_bot": True}, } - self.comments, self.calls, self.ref = list(comments), [], ref + self.comments, self.calls = list(comments), [] def __call__(self, *args, token=None): self.calls.append(args) @@ -109,10 +109,6 @@ def __call__(self, *args, token=None): return json.dumps(self.pr) if args[:2] == ("api", "--paginate"): return "\n".join(json.dumps(c) for c in self.comments) - if args[:2] == ("api", "repos/o/r/git/ref/heads/backport/0.187/ccccccc"): - if not self.ref: - raise RuntimeError("gh api (exit 1): Not Found (HTTP 404)") - return json.dumps({"object": {"sha": self.ref}}) if args[:4] == ("api", "-X", "POST", "repos/o/r/git/commits"): return json.dumps({"sha": "e" * 40}) return "" @@ -123,7 +119,6 @@ def writes(self): a[:2] if a[0] == "pr" else a[1:4] for a in self.calls if a[:2] not in (("pr", "view"), ("api", "--paginate"), ("api", "user")) - and not a[1].startswith("repos/o/r/git/ref/") ] def said(self): @@ -265,30 +260,25 @@ def test_lost_trailer_fails_loudly(self): self.assertTrue(bm.gh.said()[0].startswith("Bot could not record the decline")) def test_head_without_a_pick_declines_nothing(self): - # A head moved onto the base still owes its fix: no decline or label; the bot deletes the head, which would - # fail every later Backport run, and says once how to decline by hand. The run passes. + # A head moved onto the base still owes its fix: no decline, label or delete (removing the head is manual); + # one comment says so and how to decline by hand. The run passes. bm.first_pick = lambda pr: (None, []) - delete = ("-X", "DELETE", "repos/o/r/git/refs/heads/backport/0.187/ccccccc") self.assertEqual( self.close("CLOSED", labels=["backport-conflict"]), "skipped: no pick" ) - self.assertEqual(bm.gh.writes(), [delete, ("pr", "comment")]) + self.assertEqual(bm.gh.writes(), [("pr", "comment")]) said = bm.gh.said()[0] self.assertTrue( said.startswith(bm.NO_PICK + " release/0.187, so nothing was declined") ) + self.assertIn("Delete `backport/0.187/ccccccc` by hand", said) self.assertIn("Declined: ", said) out = self.close("CLOSED", comments=[comment(said, user="bot")]) # a re-run - self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [delete])) - # A re-run after the next Backport run re-picked onto the same name, or after a person deleted the head: - # that head is not this PR's, so it stays. - for ref in ("b" * 40, None): - out = self.close("CLOSED", comments=[comment(said, user="bot")], ref=ref) - self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [])) + self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [])) # A comment from an earlier close does not count for this one. early = comment(said, at="2026-10-12T09:00:00Z", user="bot") self.close("CLOSED", comments=[early]) - self.assertEqual(bm.gh.writes(), [delete, ("pr", "comment")]) + self.assertEqual(bm.gh.writes(), [("pr", "comment")]) def test_reopened_sets_conflict_only(self): self.assertEqual(self.close("OPEN", labels=["backport-declined"]), "reopened") @@ -423,8 +413,8 @@ def test_blocks(self): [ "**Backport PR** #246 on `release/0.189`: **skipped**: its head has no commit past the base, " "so nothing was declined", - "Next: the next Backport run treats the fix as not picked; the PR comment says how to decline " - "it by hand.", + "Next: delete the head by hand, or re-pick onto it; the PR comment says how to decline the fix " + "by hand.", ], ) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index c949c099cd..4df26275c3 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -16,8 +16,8 @@ run-name: >- # or close it. The bot records what happened (backport_merge.py --pr N): # merged -> label backport-merged, head deleted; closed unmerged -> a decline: empty decline commit on the base # (the closer's close comment is the reason), label backport-declined, head kept so the PR can be reopened; -# a head with no commit past the base declines nothing: the bot deletes it while it is still the PR's commit and -# comments how to decline by hand; +# a head with no commit past the base declines and deletes nothing: the bot comments once that a person deletes the +# head or re-picks onto it, and how to decline by hand; # reopened -> label backport-conflict. Reopen and merge undoes a decline: the newest trailer on the line wins. # pull_request_target, not pull_request: GitHub runs no pull_request workflow on a PR that conflicts with its base # at the git level (no merge ref), and a conflict PR whose base later changed the same lines is one. From 5d627574ef6df058b2d54d486c762b6283fb044e Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 16:23:46 +0530 Subject: [PATCH 20/37] fix: say how to recover a kept no-pick head Delete the head to have the fix picked again, or keep it until a hand-made decline is on the base; the delete line is posted only while the head still holds the PR's commit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 16 +++++++-- .github/release/backport_merge.py | 48 ++++++++++++++++++++------ .github/release/test_backport.py | 13 +++++++ .github/release/test_backport_merge.py | 38 ++++++++++++++++++-- .github/workflows/backport.yaml | 2 +- 5 files changed, 100 insertions(+), 17 deletions(-) diff --git a/.github/release/backport.py b/.github/release/backport.py index ff82a3cacd..3622216bd8 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -8,7 +8,8 @@ only on stabilized ones (accept=HELD, held for the RM without --confirm-stabilized). A run cancelled by a newer push costs nothing: the next run rescans. The exception is a run stopped after pushing a pick head and before opening its PR, or whose `gh pr create` failed: after PICK_TIMEOUT that head fails later runs until a person deletes it -(finish_open). +(finish_open). So does a pick PR closed with no commit past its base, whose head the record job keeps. Keeping the +run red until a person acts is deliberate: a passing run would hide a fix that is neither picked nor declined. A decline carries the pick trailer, so a declined fix is never re-picked; an open conflict PR is left alone (`skipped: conflict PR pending`) until a person merges or closes it; an open clean pick PR whose merge failed in an earlier run is merged by the next run. @@ -741,7 +742,10 @@ def finish_open(slug, branch, head): Known limits: the trailer is checked, not the commit's content, so a writer who force-pushes the bot's open pick PR with one commit carrying that trailer gets it squashed (a writer can already run a workflow with the App's token). A head whose PR was closed but whose decline record failed also reads as "no open PR": deleting that - head re-picks the declined fix; re-run the record job instead. + head re-picks the declined fix; re-run the record job instead. A closed pick PR whose head has no commit past + the base also fails every run until the head is deleted (deliberately, see the module docstring). The age is + the head tip's commit time, so a head moved onto a fresh base tip reads as "another run is picking it" for up + to PICK_TIMEOUT. """ pending = "skipped", f"conflict PR pending on `{head}`" prs = json.loads( @@ -778,9 +782,15 @@ def finish_open(slug, branch, head): "skipped", f"head {head} is {int(age // 60)} min old with no open PR: another run is picking it", ) + # A head with no commit past the base: reopening its PR only parks it as a conflict PR with nothing to merge. + empty = ( + rm.git(REPO, "rev-list", "--count", f"origin/{branch}..origin/{head}") == "0" + ) return ( "failed", - f"head {head} exists with no open PR: delete it or reopen its PR, then rerun", + f"head {head} exists with no open PR: delete it" + + ("" if empty else " or reopen its PR") + + ", then rerun", ) if len(prs) != 1: return pending diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index fe2abeca38..815db6402d 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -17,8 +17,9 @@ label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. A head with no commit past the - base (moved onto it) -> nothing declined, no label and no delete: the bot comments once that the head is to be - deleted by hand or re-picked onto, and how to decline the fix by hand. + base (moved onto it) -> nothing declined, no label and no delete: the bot comments once. While the head is still + this PR's, the comment says to delete it by hand, or to keep it until a hand-made decline is on the base; once + the head is gone or holds another pick, it says only that nothing was declined. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. @@ -200,6 +201,16 @@ def delete_head(slug, head): raise +def head_ref(slug, head): + """The sha the head branch points at, or None when it is gone.""" + try: + return gh("api", f"repos/{slug}/git/ref/heads/{head}", "--jq", ".object.sha") + except RuntimeError as e: + if "404" not in str(e): + raise + return None + + def comments(slug, n): out = gh( "api", @@ -326,9 +337,10 @@ def record(slug, n, sender): or f"Closed by @{sender} without a reason (PR #{n})" ) sha, written = decline(slug, pr, reason) - # No pick on the head: nothing to decline. Removing the head is manual: until a person deletes it (or re-picks - # onto it), every Backport run fails on it. + # No pick on the head: nothing to decline. Removing the head is manual: until a person deletes it, every + # Backport run fails on it. The head may already be gone (deleting it closes the PR) or hold a later run's pick. if not sha: + kept = head_ref(slug, head) == pr["headRefOid"] if not any( c["body"].startswith(NO_PICK) and c["created_at"] >= pr["closedAt"] for c in cs @@ -340,12 +352,23 @@ def record(slug, n, sender): "-R", slug, "--body", - f"{NO_PICK} {base}, so nothing was declined. Delete `{head}` by hand, or re-pick the fix onto it; " - "until then Backport fails on it. To decline the fix, ask the release manager for a hand-made empty " - "commit with the pick's message, its `(cherry picked from commit
)` line and a final " - "`Declined: ` line.", + f"{NO_PICK} {base}, so nothing was declined. " + + ( + f"Until `{head}` is deleted, Backport fails on it. To have the fix picked again, delete " + f"`{head}` by hand. To decline the fix instead, do not delete the head yet: ask the release " + f"manager for a hand-made empty commit on {base} with the pick's message, its `(cherry picked " + "from commit
)` line and a final `Declined: ` line. Once it is on " + f"{base}, Backport stops failing and the head can be deleted." + if kept + else f"`{head}` no longer holds this PR's head, so there is nothing to delete here." + ), ) - return {"result": "skipped: no pick", "base": base, "sha": None, "reason": None} + return { + "result": "skipped: no pick" + ("" if kept else ", head gone"), + "base": base, + "sha": None, + "reason": None, + } set_label(slug, pr, DECLINED) if not any( c["body"].startswith("Recorded as a decline") @@ -392,7 +415,12 @@ def summary(n, sender, rec): return [ head + "**skipped**: its head has no commit past the base, so nothing was declined", - "Next: delete the head by hand, or re-pick onto it; the PR comment says how to decline the fix by hand.", + ( + "Next: delete the head by hand to have the fix picked again; to decline it, keep the head until " + "the release manager's decline commit is on the base (the PR comment says how)." + if rec["result"] == "skipped: no pick" + else "Next: nothing here; its head no longer holds this PR's commit." + ), ] return [head + f"**skipped**: already on `{base}` as {rm.commit_link(sha)}"] diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index 8e6ccf9e93..7435e5031e 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -805,6 +805,19 @@ def test_head_without_a_pr_fails(self): "or reopen its PR, then rerun", ), ) + # A head moved onto the release tip: reopening its PR would park it with nothing to merge. + self.g( + "push", "-q", "-f", "origin", f"{self.oid}^:refs/heads/backport/1.0/aaaaaaa" + ) + with mock.patch.object(bp, "PICK_TIMEOUT", 0): + out = self.finish(prs=[]) + self.assertEqual( + out, + ( + "failed", + "head backport/1.0/aaaaaaa exists with no open PR: delete it, then rerun", + ), + ) def test_not_the_bots_pick_is_not_squashed(self): # The squash bypasses the release ruleset: a person's PR on a pick head, or a commit that is not the pick diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index 4a7c3fcafd..f995c6484f 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -102,6 +102,7 @@ def __init__(self, state, labels=(), comments=()): "author": {"login": BOT, "is_bot": True}, } self.comments, self.calls = list(comments), [] + self.head_ref = "a" * 40 # the head branch still at this PR's head def __call__(self, *args, token=None): self.calls.append(args) @@ -109,6 +110,14 @@ def __call__(self, *args, token=None): return json.dumps(self.pr) if args[:2] == ("api", "--paginate"): return "\n".join(json.dumps(c) for c in self.comments) + if args[:2] == ("api", "repos/o/r/git/ref/heads/backport/0.187/ccccccc"): + if isinstance(self.head_ref, Exception): + raise self.head_ref + if self.head_ref is None: + raise RuntimeError( + "gh api repos/o/r/git/ref/heads/... (exit 1): gh: Not Found (HTTP 404)" + ) + return self.head_ref if args[:4] == ("api", "-X", "POST", "repos/o/r/git/commits"): return json.dumps({"sha": "e" * 40}) return "" @@ -119,6 +128,7 @@ def writes(self): a[:2] if a[0] == "pr" else a[1:4] for a in self.calls if a[:2] not in (("pr", "view"), ("api", "--paginate"), ("api", "user")) + and "/git/ref/heads/" not in a[1] ] def said(self): @@ -271,7 +281,8 @@ def test_head_without_a_pick_declines_nothing(self): self.assertTrue( said.startswith(bm.NO_PICK + " release/0.187, so nothing was declined") ) - self.assertIn("Delete `backport/0.187/ccccccc` by hand", said) + self.assertIn("delete `backport/0.187/ccccccc` by hand", said) + self.assertIn("do not delete the head yet", said) self.assertIn("Declined: ", said) out = self.close("CLOSED", comments=[comment(said, user="bot")]) # a re-run self.assertEqual((out, bm.gh.writes()), ("skipped: no pick", [])) @@ -279,6 +290,23 @@ def test_head_without_a_pick_declines_nothing(self): early = comment(said, at="2026-10-12T09:00:00Z", user="bot") self.close("CLOSED", comments=[early]) self.assertEqual(bm.gh.writes(), [("pr", "comment")]) + # The head already deleted (that closed the PR), or holding a later run's pick: no delete advice. + for ref in (None, "d" * 40): + bm.gh = Gh("CLOSED") + bm.gh.head_ref = ref + self.assertEqual(bm.close("o/r", 7, ME), "skipped: no pick, head gone") + self.assertEqual(bm.gh.writes(), [("pr", "comment")]) + said = bm.gh.said()[0] + self.assertIn("no longer holds this PR's head", said) + self.assertNotIn("delete `", said) + # A ref read that fails for another reason (auth, outage) is not read as "head gone". + bm.gh = Gh("CLOSED") + bm.gh.head_ref = RuntimeError( + "gh api repos/o/r/git/ref/heads/... (exit 1): gh: Bad credentials (HTTP 401)" + ) + with self.assertRaisesRegex(RuntimeError, "HTTP 401"): + bm.close("o/r", 7, ME) + self.assertEqual(bm.gh.writes(), []) def test_reopened_sets_conflict_only(self): self.assertEqual(self.close("OPEN", labels=["backport-declined"]), "reopened") @@ -413,10 +441,14 @@ def test_blocks(self): [ "**Backport PR** #246 on `release/0.189`: **skipped**: its head has no commit past the base, " "so nothing was declined", - "Next: delete the head by hand, or re-pick onto it; the PR comment says how to decline the fix " - "by hand.", + "Next: delete the head by hand to have the fix picked again; to decline it, keep the head " + "until the release manager's decline commit is on the base (the PR comment says how).", ], ) + self.assertEqual( + bm.summary(246, ME, rec("skipped: no pick, head gone", sha=None))[1], + "Next: nothing here; its head no longer holds this PR's commit.", + ) class SquashRetry(unittest.TestCase): diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 4df26275c3..438732521c 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -17,7 +17,7 @@ run-name: >- # merged -> label backport-merged, head deleted; closed unmerged -> a decline: empty decline commit on the base # (the closer's close comment is the reason), label backport-declined, head kept so the PR can be reopened; # a head with no commit past the base declines and deletes nothing: the bot comments once that a person deletes the -# head or re-picks onto it, and how to decline by hand; +# head to have the fix picked again, or keeps it until a hand-made decline is on the base; # reopened -> label backport-conflict. Reopen and merge undoes a decline: the newest trailer on the line wins. # pull_request_target, not pull_request: GitHub runs no pull_request workflow on a PR that conflicts with its base # at the git level (no merge ref), and a conflict PR whose base later changed the same lines is one. From adfcbd9ab4dc1da50e1f0ddef0ff8d0253a8f075 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 16:25:33 +0530 Subject: [PATCH 21/37] feat: lint PR titles and labels on fork PRs too Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/lint-pr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index 78f10644eb..6924c56547 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -9,7 +9,7 @@ name: Lint PR # Label `release-branch-only` marks a change that lives only on its release branch: refused on main. On a release # branch, Release branch CI's bot commit writes `Release-branch-only: ` (or `#` without a ticket), which # Squash and merge keeps in the message. -# A PR from a fork is not checked, title or labels: the maintainer checks both before merging. +# A PR from a fork is checked too: the run reads only the event payload and main's relmgr, with a read-only token. on: pull_request: branches: [main, 'release/**'] @@ -20,7 +20,6 @@ permissions: jobs: lint-pr: - if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 20 steps: From 4c1e86da3023bd11d9e1b03156cafcee0d474987 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 27 Oct 2026 16:53:26 +0530 Subject: [PATCH 22/37] fix: read a chart push tagged without a v Mirrors the shared release tooling: Publish reads the registry through relstate._read, and the OSS workflow tests stub registry reads in-process. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/relstate.py | 23 +++++++++++++++++---- .github/release/test_oss.py | 34 +++++++++++++++++++++++--------- .github/release/test_relstate.py | 30 +++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/.github/release/relstate.py b/.github/release/relstate.py index ec7c402274..8068147307 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -162,6 +162,19 @@ def registry_tags(artifact): return out +def _read(artifact, registry): + """registry(artifact), with a chart push's X.Y.Z tag read as vX.Y.Z. A chart push is tagged with its Chart.yaml + version, which has no v; pushes before the release-line flow were tagged vX.Y.Z and read as they are. + """ + tags = registry(artifact) + if not artifact.startswith("chart/"): + return tags + return { + ("v" + t if re.fullmatch(r"\d+\.\d+\.\d+", t) else t): info + for t, info in tags.items() + } + + def taken_tags(reg=None, registry=registry_tags): """Every tag on every registry artifact of TARGETS: an RC there is never allocated again (rc_version taken). reg: {artifact: tags} already read; artifacts missing from it are read. @@ -268,6 +281,8 @@ def target_state(t, name, rows, now): while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. """ ga = rows[name]["targets"][t]["ga"] + # Known limit: a target built but never GA (on-prem RCs with no chart push) stays stabilizing for good, so its + # line never reaches the retire list; a person retires it with Retire `retire_line`. if not ga: return "stabilizing" if t in KEPT: @@ -323,7 +338,7 @@ def _next_minor(repo, ordered): def derive(repo, now, registry=registry_tags, rc=None): t0 = time.monotonic() reg = { - a: git_tags(repo) if a == GIT else registry(a) + a: git_tags(repo) if a == GIT else _read(a, registry) for a in sorted({a for pair in TARGETS.values() for a in pair}) } print( @@ -396,8 +411,8 @@ def derive(repo, now, registry=registry_tags, rc=None): if not tt: continue tt["rcs"].sort(key=_vkey) - # On-prem GA reads the chart artifact: RC chart pushes are tagged 0.0.0-rc.N and match no line; the - # stable chart push is tagged vX.Y.Z (push-charts-to-gcr.yaml checks it against the release tag). + # On-prem GA reads the chart artifact. Publish pushes the stable chart as X.Y.Z, read as vX.Y.Z (_read); + # RC charts are never pushed, and legacy 0.0.0-rc.N pushes match no line. stable = re.compile(rf"{re.escape(PREFIX)}v{re.escape(row['line'])}\.\d+") # ga (the anchor's first stable) anchors state and the support window; promoted (the build artifact's # latest stable) is what ships next @@ -486,7 +501,7 @@ def abandon_plan(repo, branch, registry=registry_tags): x, y = rm._line(branch) rc_re = re.compile(rf"{re.escape(PREFIX)}v{x}\.{y}\.\d+-rc\.\d+") reg = { - a: registry(a) + a: _read(a, registry) for a in sorted({a for pair in TARGETS.values() for a in pair} - {GIT}) } main = _main(repo) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 5c232ab57a..14ce961394 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -601,9 +601,10 @@ def inline(workflow, step): "the cloud repo's workflows", ) class Workflow(unittest.TestCase): - """A cloud and an OSS origin with clones; a stub gh serves the cloud registry (reg.jsonl), the open - backport-conflict PRs of each repo (_.json; a `prfail` file fails the listing) and the PRs of a main - commit (pulls.json). gh.log: each pr list and commit PRs call, as its token. + """A cloud and an OSS origin with clones; a sitecustomize stands in for Artifact Registry with reg.jsonl as + cloud/backend's versions (other artifacts empty); a stub gh serves the open backport-conflict PRs of each repo + (_.json; a `prfail` file fails the listing) and the PRs of a main commit (pulls.json). gh.log: + each pr list and commit PRs call, as its token. """ def setUp(self): @@ -620,12 +621,26 @@ def setUp(self): self.repos[name] = (origin, work) os.makedirs(os.path.join(self.cloud, ".relmgr", ".github")) os.symlink(HERE, os.path.join(self.cloud, ".relmgr", ".github", "release")) + self.site = os.path.join(d, "site") + os.mkdir(self.site) + with open(os.path.join(self.site, "sitecustomize.py"), "w") as fh: + fh.write(f"""import json, os, sys +sys.path.insert(0, {HERE!r}) +import relstate +def _ar_versions(artifact): # reg.jsonl as Artifact Registry versions of cloud/backend; other artifacts empty + if artifact != "cloud/backend" or not os.path.exists({os.path.join(d, "reg.jsonl")!r}): + return [] + with open({os.path.join(d, "reg.jsonl")!r}) as fh: + rows = [json.loads(line) for line in fh] + return [{{"name": f"r/packages/backend/versions/{{v['digest']}}", "createTime": v["pushed_at"], + "relatedTags": [{{"name": f"r/packages/backend/tags/{{t}}"}} for t in v["tags"]]}} for v in rows] +relstate._ar_versions = _ar_versions +""") bin_ = os.path.join(d, "bin") os.mkdir(bin_) - with open(os.path.join(bin_, "gh"), "w") as fh: # other artifacts empty + with open(os.path.join(bin_, "gh"), "w") as fh: fh.write(f"""#!/bin/sh case "$*" in - *%2Fcloud%2Frelease/versions*) cat {d}/reg.jsonl 2>/dev/null;; "pr list -R "*) echo "$GH_TOKEN $*" >> {d}/gh.log; [ -e {d}/prfail ] && exit 1 cat "{d}/$(echo "$4" | tr / _).json" 2>/dev/null || echo "[]";; *"/pulls"*) echo "$GH_TOKEN $*" >> {d}/gh.log; [ -e {d}/pullsfail ] && exit 1 @@ -664,6 +679,7 @@ def run_step(self, script, **env): env = { **os.environ, "PATH": self.path, + "PYTHONPATH": self.site, "FORCE": "false", "REASON": "", "OSS_REPO": self.repos["oss"][1], @@ -991,8 +1007,8 @@ def test_allocates_past_a_registry_rc(self): class PublishWorkflow(Workflow): - """Publish's Plan reads the OSS pins of the RC and the previous stable; its Changelog step points the fetcher at - this repo and the OSS repo, and swaps in the bot token for the OSS call only. + """Publish's Plan reads the OSS pins of the RC and the previous stable; its Changelog step runs the fetcher + against this repo and the OSS repo with one token (the release bot's, which reads both). """ def release( @@ -1050,7 +1066,7 @@ def test_a_side_with_no_pin_skips_oss(self): (out["prev_oss"], out["oss"]), ("ee-v1.7.0-rc.1", "ee-v1.7.0-rc.1") ) # same pin: skipped - def test_changelog_reads_both_repos_with_the_bot_token_for_oss_only(self): + def test_changelog_reads_both_repos_with_one_token(self): # The real fetcher with a stand-in requests that logs each call and a main that calls it. action = os.path.join(self.cloud, ".github", "actions", "update-changelogs") os.makedirs(action) @@ -1097,7 +1113,7 @@ def main(): self.read("calls").splitlines(), [ "Zipstack/unstract-cloud token workflow-token", - "Zipstack/unstract token oss-token", + "Zipstack/unstract token workflow-token", "Zipstack/unstract-cloud token workflow-token", ], ) diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index eefbe7f17e..c1fa7cd0f7 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -1,4 +1,4 @@ -"""Synthetic-repo tests for relstate (derive 1-11, on-prem newest six 7b, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ticketless titles 18c, Release-branch-only key 18d, release-branch-only label 18e, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, no order block on a GA line 23b, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37, Artifact Registry reads 38-41 and 39b). Run: python test_relstate.py. release-tooling-tests.yaml runs all six on a pull request +"""Synthetic-repo tests for relstate (derive 1-11, on-prem newest six 7b, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ticketless titles 18c, Release-branch-only key 18d, release-branch-only label 18e, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, no order block on a GA line 23b, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37, Artifact Registry reads 38-41 and 39b, a chart push without v 42). Run: python test_relstate.py. release-tooling-tests.yaml runs all six on a pull request that changes this directory. """ @@ -719,6 +719,34 @@ def test_17_abandon_plan_refusals(self): human = self.commit("human RBO-less work", "release/1.9") self.assertEqual(plan()["used"], [human[:12]]) + def test_42_chart_push_without_v(self): + # Publish pushes the chart as its Chart.yaml version, X.Y.Z: it anchors on-prem GA as vX.Y.Z. + self.s3() + self.push("chart/unstract-platform", "1.5.1") + tt = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] + self.assertEqual( + (tt["state"], tt["ga"]), + ( + "stabilized", + {"tag": "v1.5.1", **self.reg["chart/unstract-platform"]["1.5.1"]}, + ), + ) + self.push("cloud/backend", "1.5.1") # only a chart tag is read this way + self.assertEqual( + self.branch(self.state(), "release/1.5")["targets"]["cloud"]["promoted"][ + "tag" + ], + "v1.5.0", + ) + self.built_by_mistake() + self.push("chart/unstract-platform", "1.9.0") + self.assertEqual( + rs.abandon_plan(self.repo, "release/1.9", lambda a: self.reg.get(a, {}))[ + "ga" + ], + ["chart/unstract-platform:v1.9.0"], + ) + def test_26_oss_repo_config(self): # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), # supported while its branch exists once its on-prem-ga/X.Y tag is there. Plain v* tags are community releases from main and never count. From 869fbf1a37364eb306c4fd048e044b1159a94846 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 17:00:38 +0530 Subject: [PATCH 23/37] feat: mark a stabilized OSS line with stabilized/X.Y Replaces the on-prem-ga/X.Y tag: cloud Promote writes stabilized/X.Y in the promote that leaves every enabled cloud target GA (oss.py promote --stabilized). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/oss.py | 30 ++++++++++++------------ .github/release/relstate.py | 24 ++++++++++++++------ .github/release/test_oss.py | 39 ++++++++++++++++++++++---------- .github/release/test_relstate.py | 28 +++++++++++++++++++---- 4 files changed, 84 insertions(+), 37 deletions(-) diff --git a/.github/release/oss.py b/.github/release/oss.py index 0ce67e7366..5b62845e0b 100644 --- a/.github/release/oss.py +++ b/.github/release/oss.py @@ -5,10 +5,10 @@ oss.py cut --line X.Y [--slack-url URL] cut OSS release/X.Y from OSS main; prints ee-vX.Y.0-rc.1 oss.py rc --branch release/X.Y [--slack-url URL] the tag to pin for the next RC of a cloud line: the tag on its OSS line's tip, after a new OSS RC when the tip has none - oss.py promote --rc vX.Y.Z-rc.N [--force --reason R] [--on-prem] + oss.py promote --rc vX.Y.Z-rc.N [--force --reason R] [--stabilized] promote the OSS RC the cloud RC pins; prints the stable tag. - --on-prem (the cloud line's on-prem promote) also tags the - OSS line on-prem-ga/X.Y once, which makes it stabilized + --stabilized (the promote that leaves every enabled cloud + target GA) also tags the OSS line stabilized/X.Y once retire(): called by cloud Retire for each cloud line it retires. cut, rc and promote print one OSS tag on stdout. Every write is re-runnable: a tag already on the same commit is @@ -247,15 +247,16 @@ def rc(oss, branch, slack_url=""): # Known limit: force accepts an empty reason. -def promote(repo, oss, cloud_rc, force=False, reason="", on_prem=False): - """The stable OSS tag of the OSS RC cloud_rc pins (_promote). on_prem: the cloud line's on-prem promote, so the - OSS line's on-prem-ga/X.Y tag goes on that stable tag's commit when the line has none (a later patch keeps it). - Known limit: the tag marks the promote, not the chart push after it; a failed chart push leaves the OSS line - stabilized before its cloud line's on-prem GA. +def promote(repo, oss, cloud_rc, force=False, reason="", stabilized=False): + """The stable OSS tag of the OSS RC cloud_rc pins (_promote). stabilized: this promote leaves every enabled + target of the cloud line GA, so the OSS line's stabilized/X.Y tag goes on that stable tag's commit when the line + has none (a later promote keeps it). + Known limit: the tag marks the promote, not the retag or chart push after it; if those fail, the OSS line is + stabilized before its cloud line is GA. """ stable = _promote(repo, oss, cloud_rc, force, reason) - mark = rs.ON_PREM_GA + line_of(stable).split("/", 1)[1] - if on_prem and not rs._tag_commit(oss, mark): + mark = rs.STABILIZED + line_of(stable).split("/", 1)[1] + if stabilized and not rs._tag_commit(oss, mark): sha = rs._tag_commit(oss, stable) if not sha: raise RuntimeError(f"no OSS tag {stable}") @@ -263,7 +264,7 @@ def promote(repo, oss, cloud_rc, force=False, reason="", on_prem=False): oss, mark, sha, - [f"on-prem GA: {stable} (cloud {cloud_rc})"], + [f"stabilized: {stable} (cloud {cloud_rc})"], f"refs/tags/{mark}", ) return stable @@ -335,7 +336,8 @@ def _promote(repo, oss, cloud_rc, force, reason): def retire(oss, branch, reason, dry_run=True): """Retire OSS release/X.Y with its cloud line, in one atomic push: archive tag retired/X.Y-, delete - (no OSS GA) its RC tags and the branch. With a GA its tags stay and the minor is never reused; without one it is. Summary lines. + (no OSS GA) its RC tags and the branch. With a GA its tags stay, stabilized/X.Y with them, and the minor is never + reused; without one it is. Summary lines. """ line = branch.removeprefix("release/") sha = tip(oss, branch) @@ -406,7 +408,7 @@ def main(argv=None): m.add_argument("--rc", required=True) m.add_argument("--force", action="store_true") m.add_argument("--reason", default="") - m.add_argument("--on-prem", action="store_true") + m.add_argument("--stabilized", action="store_true") a = p.parse_args(argv) try: if not a.oss_repo: @@ -416,7 +418,7 @@ def main(argv=None): if a.cmd == "cut" else rc(a.oss_repo, a.branch, a.slack_url) if a.cmd == "rc" - else promote(".", a.oss_repo, a.rc, a.force, a.reason, a.on_prem) + else promote(".", a.oss_repo, a.rc, a.force, a.reason, a.stabilized) ) except RuntimeError as e: print(f"::error::{e}", file=sys.stderr) # stdout is the tag the workflow captures diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 8068147307..8a04016ce8 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -74,8 +74,9 @@ def use(slug): AR_API = "https://artifactregistry.googleapis.com/v1" -# OSS tag on-prem-ga/X.Y: the paired cloud line reached on-prem GA. OSS git cannot see the cloud registry. -ON_PREM_GA = "on-prem-ga/" +# OSS tag stabilized/X.Y: every enabled target of the paired cloud line reached GA. OSS git cannot see the cloud +# registry, so cloud Promote writes it. +STABILIZED = "stabilized/" def ar_repository(kind): @@ -276,9 +277,9 @@ def target_state(t, name, rows, now): On-prem: stabilized for WINDOW after the line's first stable chart push (a patch does not extend it), and only while among the newest WINDOW_LINES lines with an on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch - for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilizing until its - paired cloud line's on-prem GA (the ON_PREM_GA tag, which `oss.py promote --on-prem` writes), then stabilized - while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. + for it stays stabilized; older GA branches are past-eos for it. A KEPT target (OSS): stabilizing until every + enabled target of its paired cloud line is GA (the STABILIZED tag, which `oss.py promote --stabilized` writes), + then stabilized while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. """ ga = rows[name]["targets"][t]["ga"] # Known limit: a target built but never GA (on-prem RCs with no chart push) stays stabilizing for good, so its @@ -286,7 +287,7 @@ def target_state(t, name, rows, now): if not ga: return "stabilizing" if t in KEPT: - return "stabilized" if rows[name]["targets"][t]["on_prem_ga"] else "stabilizing" + return "stabilized" if rows[name]["targets"][t]["stabilized"] else "stabilizing" t_ga = sorted( (n for n, r in rows.items() if r["targets"].get(t, {}).get("ga")), key=lambda n: _key(rows[n]["line"]), @@ -422,7 +423,7 @@ def derive(repo, now, registry=registry_tags, rc=None): ): tt[key] = {"tag": tags[pick], **reg[artifact][tags[pick]]} if t in KEPT: - tt["on_prem_ga"] = bool(_tag_commit(repo, ON_PREM_GA + row["line"])) + tt["stabilized"] = bool(_tag_commit(repo, STABILIZED + row["line"])) for name, row in rows.items(): for t, tt in row["targets"].items(): @@ -553,6 +554,15 @@ def abandon_plan(repo, branch, registry=registry_tags): } +def stabilizes(row, target, done=()): + """Whether promoting target leaves every enabled target of the line row GA: each is target, in done (promoted + earlier in the same run), or already GA. Cloud Promote then marks the pinned OSS line stabilized. + """ + return all( + u == target or u in done or row["targets"][u]["ga"] for u in row["enabled"] + ) + + def promote_blockers(state, branch, target): """Older lines not GA for `target` yet, one refusal line each: Promote of `branch` for `target` waits for them. diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 14ce961394..2861075c34 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -218,26 +218,28 @@ def test_promote(self): with self.assertRaisesRegex(RuntimeError, "did not run"): oss.promote(self.cloud, self.oss, "v1.7.1-rc.2", force=True, reason="x") - def test_on_prem_promote_tags_on_prem_ga_once(self): - # The OSS line stays stabilizing until its cloud line's on-prem GA: only --on-prem writes the tag, once. + def test_stabilized_promote_tags_the_line_once(self): + # The OSS line stays stabilizing until its cloud line is GA on every enabled target: only --stabilized + # writes the tag, once; a later promote keeps it on the GA commit. self.quiet(oss.cut, self.oss, "1.7") self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") - oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") # the cloud product - self.assertEqual(self.remote("tag", "-l", "on-prem-ga/*"), "") + # Cloud GA, on-prem still enabled. + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1") + self.assertEqual(self.remote("tag", "-l", "stabilized/*"), "") self.clone() - oss.promote(self.cloud, self.oss, "v1.7.0-rc.1", on_prem=True) + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1", stabilized=True) ga = self.remote("rev-parse", "ee-v1.7.0^{commit}") - self.assertEqual(self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), ga) + self.assertEqual(self.remote("rev-parse", "stabilized/1.7^{commit}"), ga) self.work("release/1.7", "oss fix") self.quiet(oss.rc, self.oss, "release/1.7") self.pin("ee-v1.7.1-rc.1", "v1.7.1-rc.1") self.clone() self.assertEqual( - oss.promote(self.cloud, self.oss, "v1.7.1-rc.1", on_prem=True), "ee-v1.7.1" + oss.promote(self.cloud, self.oss, "v1.7.1-rc.1", stabilized=True), "ee-v1.7.1" ) - self.assertEqual(self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), ga) + self.assertEqual(self.remote("rev-parse", "stabilized/1.7^{commit}"), ga) - def test_cli_on_prem_flag(self): + def test_cli_stabilized_flag(self): self.quiet(oss.cut, self.oss, "1.7") self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") self.addCleanup(os.chdir, os.getcwd()) @@ -245,11 +247,11 @@ def test_cli_on_prem_flag(self): out = io.StringIO() with contextlib.redirect_stdout(out): code = oss.main( - ["--oss-repo", self.oss, "promote", "--rc", "v1.7.0-rc.1", "--on-prem"] + ["--oss-repo", self.oss, "promote", "--rc", "v1.7.0-rc.1", "--stabilized"] ) self.assertEqual((code, out.getvalue()), (0, "ee-v1.7.0\n")) self.assertEqual( - self.remote("rev-parse", "on-prem-ga/1.7^{commit}"), + self.remote("rev-parse", "stabilized/1.7^{commit}"), self.remote("rev-parse", "ee-v1.7.0^{commit}"), ) @@ -827,7 +829,13 @@ def prs(self, slug, *bases): ) def promote(self, rc="v1.7.0-rc.1", **env): - return self.run_step(inline("promote.yaml", "Plan"), RC=rc, TARGET="cloud", **env) + return self.run_step( + inline("promote.yaml", "Plan"), + RC=rc, + TARGET="cloud", + PRODUCTS='["cloud"]', + **env, + ) def blocked(self, r, msg): self.assertEqual(r.returncode, 1, r.stdout + r.stderr) @@ -864,6 +872,13 @@ def test_promote_conflict_prs(self): f"ship it; open conflict PRs: {oss.OSS}#40", ) + def test_promote_marks_a_cloud_only_line_stabilized(self): + # A cloud-only line: its cloud promote leaves every enabled target GA, so the OSS step gets --stabilized. + self.line("1.7", "v1.7.0-rc.1") + r = self.promote() + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("stabilized=true\n", self.read("output")) + def test_promote_fails_closed(self): self.line("1.7", "v1.7.0-rc.1") self.write("prfail", "") diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index c1fa7cd0f7..73f3470bae 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -747,9 +747,29 @@ def test_42_chart_push_without_v(self): ["chart/unstract-platform:v1.9.0"], ) + def test_43_stabilizes_when_every_enabled_target_is_ga(self): + # Cloud Promote marks the OSS line stabilized/X.Y on the promote that leaves every enabled target GA. + def row(): + return self.branch(self.state(), "release/1.5") + + self.s1() # a cloud-only line: its cloud GA + self.assertTrue(rs.stabilizes(row(), "cloud")) + self.push("on-prem/backend", "v1.5.0-rc.1") # on-prem enabled, nothing GA + self.assertEqual(row()["enabled"], ["cloud", "on-prem"]) + self.assertFalse(rs.stabilizes(row(), "cloud")) + self.assertFalse(rs.stabilizes(row(), "on-prem")) + # One run promoting both: on-prem counts the cloud promote before it. + self.assertTrue(rs.stabilizes(row(), "on-prem", ["cloud"])) + self.push("cloud/backend", "v1.5.0", git_ref="release/1.5") # cloud GA alone + self.assertFalse(rs.stabilizes(row(), "cloud")) + self.assertTrue(rs.stabilizes(row(), "on-prem")) # on-prem goes last + self.push("chart/unstract-platform", "1.5.0") + # A later patch: oss.promote keeps the tag. + self.assertTrue(rs.stabilizes(row(), "cloud")) + def test_26_oss_repo_config(self): # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), - # supported while its branch exists once its on-prem-ga/X.Y tag is there. Plain v* tags are community releases from main and never count. + # supported while its branch exists once its stabilized/X.Y tag is there. Plain v* tags are community releases from main and never count. self.addCleanup(rs.use, rs.SLUG) rs.use("unstract") self.state = lambda: rs.derive( @@ -763,13 +783,13 @@ def test_26_oss_repo_config(self): self.g("tag", f"v{line}.7", f"release/{line}") # never the line's GA self.cut("1.7") self.g("tag", "ee-v1.7.0-rc.1", "release/1.7") - # Stabilizing past its own ee-vX.Y.0 until the paired cloud line's on-prem GA tags it. - self.g("tag", "on-prem-ga/1.5", "release/1.5") + # Stabilizing past its own ee-vX.Y.0 until the paired cloud line is GA on every enabled target and tags it. + self.g("tag", "stabilized/1.5", "release/1.5") self.assertEqual( [(b["branch"], b["state"]) for b in self.state()["branches"]][1:], [("release/1.6", "stabilizing"), ("release/1.5", "stabilized")], ) - self.g("tag", "on-prem-ga/1.6", "release/1.6") + self.g("tag", "stabilized/1.6", "release/1.6") st = self.state() self.assertEqual( [(b["branch"], b["state"]) for b in st["branches"]], From 7fce0feea614a0dc538107d68009f19a420303b4 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 17:31:27 +0530 Subject: [PATCH 24/37] feat: delete stabilized/X.Y when the paired cloud line reopens An enterprise RC that builds a target with no GA yet makes the OSS line stabilizing again (oss.py reopen, relstate.reopens); Promote writes the tag again at the line's last GA. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/oss.py | 16 +++++++++++++++ .github/release/relstate.py | 7 +++++++ .github/release/test_oss.py | 35 +++++++++++++++++++++++++++++++- .github/release/test_relstate.py | 13 ++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/.github/release/oss.py b/.github/release/oss.py index 5b62845e0b..dd6f1b9a36 100644 --- a/.github/release/oss.py +++ b/.github/release/oss.py @@ -9,6 +9,7 @@ promote the OSS RC the cloud RC pins; prints the stable tag. --stabilized (the promote that leaves every enabled cloud target GA) also tags the OSS line stabilized/X.Y once +reopen(): called by cloud Take RC build after an RC that reopens the cloud line builds. retire(): called by cloud Retire for each cloud line it retires. cut, rc and promote print one OSS tag on stdout. Every write is re-runnable: a tag already on the same commit is @@ -246,6 +247,21 @@ def rc(oss, branch, slack_url=""): return tag +def reopen(oss, branch): + """Delete OSS release/X.Y's stabilized/X.Y tag: its cloud line built a target with no GA yet, so the OSS line is + stabilizing again and its next main push catches up the plain fixes it skipped. Promote writes the tag again at + the line's last GA. A missing tag is a no-op. Summary lines. + """ + mark = rs.STABILIZED + branch.removeprefix("release/") + if not rs._tag_commit(oss, mark): + return [] + _push(oss, f":refs/tags/{mark}") + rm.git(oss, "tag", "-d", mark) + return [ + f"- OSS `{branch}`: deleted `{mark}`, stabilizing until Promote's last GA tags it again" + ] + + # Known limit: force accepts an empty reason. def promote(repo, oss, cloud_rc, force=False, reason="", stabilized=False): """The stable OSS tag of the OSS RC cloud_rc pins (_promote). stabilized: this promote leaves every enabled diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 8a04016ce8..90e59ac1de 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -563,6 +563,13 @@ def stabilizes(row, target, done=()): ) +def reopens(row, built): + """Whether an RC building targets built reopens the line row: one of them has no GA on it yet (on-prem enabled + after cloud GA, say), so the line is no longer GA on every enabled target and its OSS line's STABILIZED tag goes. + """ + return not all(row["targets"].get(t, {}).get("ga") for t in built) + + def promote_blockers(state, branch, target): """Older lines not GA for `target` yet, one refusal line each: Promote of `branch` for `target` waits for them. diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 2861075c34..d5012e0b5f 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -239,6 +239,23 @@ def test_stabilized_promote_tags_the_line_once(self): ) self.assertEqual(self.remote("rev-parse", "stabilized/1.7^{commit}"), ga) + def test_reopen_deletes_the_stabilized_tag(self): + # Cloud Take RC built a target with no GA: the OSS line is stabilizing again. No tag is a no-op. + self.quiet(oss.cut, self.oss, "1.7") + self.assertEqual(oss.reopen(self.oss, "release/1.7"), []) + self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") + oss.promote(self.cloud, self.oss, "v1.7.0-rc.1", stabilized=True) + self.clone() + self.assertEqual( + oss.reopen(self.oss, "release/1.7"), + [ + "- OSS `release/1.7`: deleted `stabilized/1.7`, stabilizing until Promote's last GA tags it again" + ], + ) + self.assertEqual(self.remote("tag", "-l", "stabilized/*"), "") + self.assertEqual(self.remote("tag", "-l", "ee-v1.7.0"), "ee-v1.7.0") + self.assertEqual(oss.reopen(self.oss, "release/1.7"), []) # a rerun + def test_cli_stabilized_flag(self): self.quiet(oss.cut, self.oss, "1.7") self.pin("ee-v1.7.0-rc.1", "v1.7.0-rc.1") @@ -1017,7 +1034,23 @@ def test_allocates_past_a_registry_rc(self): ) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) self.assertEqual( - self.read("output"), "tag=v1.7.0-rc.2\nbump=true\ntargets=cloud\n" + self.read("output"), + "tag=v1.7.0-rc.2\nbump=true\ntargets=cloud\nreopen=true\n", + ) + + def test_patch_rc_on_a_ga_line_does_not_reopen(self): + # Every enabled target (cloud) is GA: a cloud patch RC keeps the OSS line's stabilized tag. + self.line("1.7", "v1.7.0") + self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + r = self.run_step( + inline("take-rc-build.yaml", "Plan version and targets"), + BRANCH="release/1.7", + TARGETS="cloud", + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + self.read("output"), + "tag=v1.7.1-rc.1\nbump=true\ntargets=cloud\nreopen=false\n", ) diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index 73f3470bae..a486220ef5 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -767,6 +767,19 @@ def row(): # A later patch: oss.promote keeps the tag. self.assertTrue(rs.stabilizes(row(), "cloud")) + def test_44_reopens_when_a_built_target_has_no_ga(self): + # Take RC deletes the OSS line's stabilized tag when it builds a target with no GA on the line. + def row(): + return self.branch(self.state(), "release/1.5") + + self.s2() # cloud GA, cloud only + self.assertFalse(rs.reopens(row(), ["cloud"])) # a cloud patch RC + self.assertTrue(rs.reopens(row(), ["on-prem"])) # on-prem enabled late + self.assertTrue(rs.reopens(row(), ["cloud", "on-prem"])) + self.push("on-prem/backend", "v1.5.1-rc.1", git_ref="release/1.5") + self.push("chart/unstract-platform", "1.5.1") # on-prem GA too + self.assertFalse(rs.reopens(row(), ["cloud", "on-prem"])) + def test_26_oss_repo_config(self): # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), # supported while its branch exists once its stabilized/X.Y tag is there. Plain v* tags are community releases from main and never count. From c665ac02a07e7059c6c716b95d5be4b26e1cec46 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 17:52:02 +0530 Subject: [PATCH 25/37] test: cover the Retire lease in the OSS workflow harness Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/test_oss.py | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index d5012e0b5f..005075941e 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -802,6 +802,57 @@ def refused(self, x_y, tag, **env): self.assertEqual(rm.git(origin, "tag", "-l", "retired/*"), tag) self.assertIn(f"release/{x_y}", rm.git(origin, "branch", "-l", "release/*")) + def lands_mid_run(self, x_y): + """A push to release/x_y lands on origin after this run read its tip: origin's hook moves the branch on the + run's first other push (an RC tag delete or the archive tag), before the branch delete. + """ + origin = self.repos["cloud"][0] + self.g(self.cloud, "commit", "-q", "--allow-empty", "-m", "late fix") + late = self.g(self.cloud, "rev-parse", "HEAD") + self.g(self.cloud, "push", "-q", "origin", "HEAD:refs/keep/late") + self.g(self.cloud, "reset", "-q", "--hard", "HEAD~1") + hook = os.path.join(origin, "hooks", "post-receive") + with open(hook, "w") as fh: + fh.write( + f"#!/bin/sh\ngrep -q refs/heads/ || git update-ref refs/heads/release/{x_y} {late}\n" + ) + os.chmod(hook, 0o755) + return late + + def kept(self, x_y, late, **env): + r = self.run_step( + inline("retire.yaml", "Derive and retire"), + **{"ABANDON": "", "RETIRE_LINE": "", "NOW": "", "DRY_RUN": "false", **env}, + ) + self.assertNotEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + rm.git(self.repos["cloud"][0], "rev-parse", f"release/{x_y}"), late + ) + + def test_abandon_keeps_a_branch_pushed_after_its_tip_was_read(self): + # Its RC tag delete is the push the hook waits for. + self.line("1.7", "v1.7.0-rc.1") + self.kept( + "1.7", + self.lands_mid_run("1.7"), + ABANDON="release/1.7", + REASON="cut by mistake", + ) + + def test_retire_line_keeps_a_branch_pushed_after_its_tip_was_read(self): + self.line("1.7") + self.kept( + "1.7", + self.lands_mid_run("1.7"), + RETIRE_LINE="release/1.7", + REASON="not shipping", + ) + + def test_derived_list_keeps_a_branch_pushed_after_its_tip_was_read(self): + self.line("1.6", "v1.6.0-rc.1", "v1.6.0") + self.line("1.7", "v1.7.0-rc.1", "v1.7.0") + self.kept("1.6", self.lands_mid_run("1.6")) + def test_retire_line_refuses_a_line_moved_after_its_archive(self): self.line("1.7") self.refused( From bf3fa08ab155d041603fb66b7d45a1512664a0c7 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 17:56:19 +0530 Subject: [PATCH 26/37] fix: treat a short-sha decline and a full-sha pick as one original picked_from resolves each -x trailer to its full sha, so the newer pick replaces a hand-written decline that used a short sha. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport_merge.py | 4 ++-- .github/release/relmgr.py | 11 +++++++++-- .github/release/test_convgate.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 815db6402d..56b50f8b7d 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -239,7 +239,7 @@ def decline(slug, pr, reason): slug, "--body", "Bot could not record the decline: the first commit's message needs exactly one `(cherry picked from " - "commit
)` line, naming the main commit of this branch's name. Decline it with a hand-made empty " + "commit )` line, naming the main commit of this branch's name. Decline it with a hand-made empty " "commit carrying that line, or ask the release manager.", ) raise RuntimeError( @@ -357,7 +357,7 @@ def record(slug, n, sender): f"Until `{head}` is deleted, Backport fails on it. To have the fix picked again, delete " f"`{head}` by hand. To decline the fix instead, do not delete the head yet: ask the release " f"manager for a hand-made empty commit on {base} with the pick's message, its `(cherry picked " - "from commit
)` line and a final `Declined: ` line. Once it is on " + "from commit )` line and a final `Declined: ` line. Once it is on " f"{base}, Backport stops failing and the head can be deleted." if kept else f"`{head}` no longer holds this PR's head, so there is nothing to delete here." diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py index 150a0b8a22..e1e2774edc 100644 --- a/.github/release/relmgr.py +++ b/.github/release/relmgr.py @@ -126,8 +126,15 @@ def _is_empty(repo, sha): def picked_from(repo, sha): - """Main originals named by the commit's `-x` trailers.""" - return PICK_RE.findall(git(repo, "log", "-1", "--format=%B", sha)) + """Main originals named by the commit's `-x` trailers, each resolved to its full sha so that callers keying on + them see a hand-written short sha and the bot's full one as the same original. A trailer that does not resolve + (unknown or ambiguous) is kept as written. + """ + out = [] + for o in PICK_RE.findall(git(repo, "log", "-1", "--format=%B", sha)): + r = git(repo, "rev-parse", "-q", "--verify", f"{o}^{{commit}}", check=False) + out.append(r.stdout.strip() if r.returncode == 0 else o) + return out def branch_range(repo, branch, main): diff --git a/.github/release/test_convgate.py b/.github/release/test_convgate.py index 80d9594cef..de0351cd68 100644 --- a/.github/release/test_convgate.py +++ b/.github/release/test_convgate.py @@ -171,6 +171,20 @@ def test_2e_pick_after_decline_ships(self): self.assertEqual(code, 0, out) self.assertIn("1 picks, 1 in the RC, 1 unclassifiable", out) + def test_2e2_pick_after_short_sha_decline_ships(self): + """A hand-written decline may name the original by a short sha; the bot's later pick names it in full. Both + are one original, so the pick still wins. + """ + self.decline(self.c[:7], "release/1.6") + self.pick(self.c, "release/1.6") + self.assertEqual( + rm.trailer_states(self.repo, "release/1.6"), {self.c: "cherry-pick"} + ) + self.assertEqual(rm.decline_records(self.repo, "release/1.6"), {}) + code, out = self.run_gate("release/1.5") + self.assertEqual(code, 0, out) + self.assertIn("1 picks, 1 in the RC, 1 unclassifiable", out) + def test_2f_pick_after_decline_past_ga_still_owed(self): """Newer is GA at a tag without the fix; its decline was undone by a later pick: older's promote waits for newer's next patch instead of reading the stale decline as converged. From f6759630ff61cb0598c773e0cdaf04ef992f10fc Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 22:51:53 +0530 Subject: [PATCH 27/37] fix: reopen on the ticked targets, not only the ones this run builds Port of the release tooling change from the enterprise repo (tree e07ef33e). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/relstate.py | 8 +++---- .github/release/test_oss.py | 45 ++++++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 90e59ac1de..2d20623a17 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -563,11 +563,11 @@ def stabilizes(row, target, done=()): ) -def reopens(row, built): - """Whether an RC building targets built reopens the line row: one of them has no GA on it yet (on-prem enabled - after cloud GA, say), so the line is no longer GA on every enabled target and its OSS line's STABILIZED tag goes. +def reopens(row, targets): + """Whether an RC for targets reopens the line row: one of them has no GA on it yet (on-prem enabled after cloud + GA, say), so the line is no longer GA on every enabled target and its OSS line's STABILIZED tag goes. """ - return not all(row["targets"].get(t, {}).get("ga") for t in built) + return not all(row["targets"].get(t, {}).get("ga") for t in targets) def promote_blockers(state, branch, target): diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 005075941e..13046a88fa 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -621,9 +621,9 @@ def inline(workflow, step): ) class Workflow(unittest.TestCase): """A cloud and an OSS origin with clones; a sitecustomize stands in for Artifact Registry with reg.jsonl as - cloud/backend's versions (other artifacts empty); a stub gh serves the open backport-conflict PRs of each repo - (_.json; a `prfail` file fails the listing) and the PRs of a main commit (pulls.json). gh.log: - each pr list and commit PRs call, as its token. + cloud/backend's versions and reg-_.jsonl as another artifact's (empty when absent); a stub gh + serves the open backport-conflict PRs of each repo (_.json; a `prfail` file fails the listing) and + the PRs of a main commit (pulls.json). gh.log: each pr list and commit PRs call, as its token. """ def setUp(self): @@ -646,10 +646,11 @@ def setUp(self): fh.write(f"""import json, os, sys sys.path.insert(0, {HERE!r}) import relstate -def _ar_versions(artifact): # reg.jsonl as Artifact Registry versions of cloud/backend; other artifacts empty - if artifact != "cloud/backend" or not os.path.exists({os.path.join(d, "reg.jsonl")!r}): +def _ar_versions(artifact): # reg.jsonl for cloud/backend, reg-_.jsonl for the others + f = os.path.join({d!r}, "reg.jsonl" if artifact == "cloud/backend" else "reg-" + artifact.replace("/", "_") + ".jsonl") + if not os.path.exists(f): return [] - with open({os.path.join(d, "reg.jsonl")!r}) as fh: + with open(f) as fh: rows = [json.loads(line) for line in fh] return [{{"name": f"r/packages/backend/versions/{{v['digest']}}", "createTime": v["pushed_at"], "relatedTags": [{{"name": f"r/packages/backend/tags/{{t}}"}} for t in v["tags"]]}} for v in rows] @@ -1104,6 +1105,38 @@ def test_patch_rc_on_a_ga_line_does_not_reopen(self): "tag=v1.7.1-rc.1\nbump=true\ntargets=cloud\nreopen=false\n", ) + def test_rerun_on_a_tagged_tip_reopens_for_a_target_already_built(self): + # An earlier run tagged v1.7.1-rc.1 for cloud and on-prem on a line GA for cloud only; its on-prem build + # pushed, its cloud build failed. A fresh run builds only cloud, yet on-prem still has no GA: reopen. + self.line("1.7", "v1.7.0") + self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + self.g( + self.cloud, "commit", "-q", "--allow-empty", "-m", "release: chart 1.7.1-rc.1" + ) + self.g(self.cloud, "tag", "v1.7.1-rc.1") + self.g( + self.cloud, + "push", + "-q", + "origin", + "HEAD:refs/heads/release/1.7", + "v1.7.1-rc.1", + ) + self.write( + "reg-on-prem_backend.jsonl", + '{"id": 1, "digest": "sha256:p", "pushed_at": "2026-09-02T00:00:00Z", "tags": ["v1.7.1-rc.1"]}\n', + ) + r = self.run_step( + inline("take-rc-build.yaml", "Plan version and targets"), + BRANCH="release/1.7", + TARGETS="cloud,on-prem", + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + self.read("output"), + "tag=v1.7.1-rc.1\nbump=false\ntargets=cloud\nreopen=true\n", + ) + class PublishWorkflow(Workflow): """Publish's Plan reads the OSS pins of the RC and the previous stable; its Changelog step runs the fetcher From 8f6ac8d8929cd27bfe9b4a2730e6c841326efa4f Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 10:18:14 +0530 Subject: [PATCH 28/37] fix: sync the release tooling and add the release-branch-only workflow - Release tooling matches the cloud copy: tidier comments, the newest RC kept on abandon, guard refusals reported as CLI errors, and tests for the runner roots and the topic-branch head check. - release-branch-only.yaml adds the Release-branch-only commit on the label, as in the cloud repo. - Backport runs on Python 3.12 without bytecode files and drops an unused token; Branch state drops one too. - Comment fixes in the release workflows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/backport.py | 514 ++++++++-------- .github/release/backport_merge.py | 120 ++-- .github/release/convgate.py | 596 ++++++++++--------- .github/release/oss.py | 100 ++-- .github/release/relmgr.py | 447 +++++++------- .github/release/relstate.py | 392 ++++++------ .github/release/test_backport.py | 121 ++-- .github/release/test_backport_merge.py | 27 +- .github/release/test_convgate.py | 139 +++-- .github/release/test_links.py | 2 - .github/release/test_oss.py | 231 ++++--- .github/release/test_relstate.py | 269 +++++---- .github/workflows/backport.yaml | 50 +- .github/workflows/lint-pr.yaml | 24 +- .github/workflows/release-branch-ci.yaml | 112 +--- .github/workflows/release-branch-only.yaml | 103 ++++ .github/workflows/release-tooling-tests.yaml | 1 + .github/workflows/state.yaml | 2 - 18 files changed, 1759 insertions(+), 1491 deletions(-) create mode 100644 .github/workflows/release-branch-only.yaml diff --git a/.github/release/backport.py b/.github/release/backport.py index 3622216bd8..d3677c06c8 100644 --- a/.github/release/backport.py +++ b/.github/release/backport.py @@ -3,30 +3,21 @@ Usage: backport.py --state state.json (--commits SHA|PR[,...] | --rescan) [--confirm-stabilized] [--branches release/X.Y[,...]] ---rescan (every push to main): ignore the pushed commits and pick, for each target, every main commit since its cut -that no pick or decline trailer there names: plain fixes too on fix targets (held accept=OWED), security/critical -only on stabilized ones (accept=HELD, held for the RM without --confirm-stabilized). A run cancelled by a newer push -costs nothing: the next run rescans. The exception is a run stopped after pushing a pick head and before opening -its PR, or whose `gh pr create` failed: after PICK_TIMEOUT that head fails later runs until a person deletes it -(finish_open). So does a pick PR closed with no commit past its base, whose head the record job keeps. Keeping the -run red until a person acts is deliberate: a passing run would hide a fix that is neither picked nor declined. -A decline carries the pick trailer, so a declined fix is never re-picked; an open conflict PR is left alone -(`skipped: conflict PR pending`) until a person merges or closes it; an open clean pick PR whose merge failed in -an earlier run is merged by the next run. +--rescan (every push to main): pick every main commit since each target's cut that no trailer there names (plain +fixes on fix targets; security/critical only on stabilized ones, held without --confirm-stabilized). A stale pick +head or a closed empty pick PR fails every run until a person deletes the head or reopens its PR (finish_open). An open conflict PR +is left alone. --commits (dispatch): only these commits. Run from a full clone whose `origin` pushes with the bot token, with GH_TOKEN set to the bot token and RELEASE_BOT_APP_SLUG set to the release App's slug. Every pick is a PR from backport/X.Y/ to release/X.Y titled with the original main PR's title (every pick is its own PR), so PR-title changelogs list it. The class comes from that PR's title and fix/ labels. -Clean pick: the bot squash-merges the PR in the same run through backport_merge.squash with the pick's own -message, so the -x trailer survives whatever the repo's squash default, and deletes -the head. No wait for the PR check: the bot is a bypass actor on the release/* ruleset, the check only -classifies commits, which an -x pick passes, and the push run of Release branch CI stays the backstop, as for -the direct pushes this replaced. If GitHub still holds the merge for a required check, the PR stays open and +Clean pick: the bot squash-merges the PR in the same run with the pick's own message (backport_merge.squash) and +deletes the head. It does not wait for the PR check: the bot bypasses the release ruleset, and the push run of +Release branch CI is the backstop. If GitHub still holds the merge for a required check, the PR stays open and the run reports it. -A rejected push of the pick head is retried (3 attempts, 5 s then 20 s apart); a final rejection is one -sentence naming the cause and the fix, in the run's ::error::, on the source PR and in the gate's MISSING row. +A rejected head push is retried (PUSH_WAITS); a final rejection is one sentence naming the cause and the fix, in the run's ::error::, on the source PR and in the gate's MISSING row. Conflict: committed with markers and left open with label backport-conflict. A person resolves it, gets one approving review and merges it, or closes it; Backport's record job (backport_merge.py) then records the merge, or the close as a decline (the close comment is the reason). Reopening and merging undoes a decline. @@ -35,6 +26,7 @@ """ import argparse +import collections import datetime import json import os @@ -49,6 +41,7 @@ import relstate as rs REPO = "." +MAIN = "origin/main" # the main ref every pick is resolved against BOT_ENV = {"GIT_COMMITTER_NAME": "releases-bot", "GIT_COMMITTER_EMAIL": rm.BOT_EMAIL} @@ -67,6 +60,18 @@ def run(*args, check=True, input=None, env=None): return r +def fetch_branch(name, check=True): + """Fetch origin's branch name into origin/.""" + return run( + "git", + "fetch", + "-q", + "origin", + f"+refs/heads/{name}:refs/remotes/origin/{name}", + check=check, + ) + + def soft(what, *args): """A best-effort command (a comment, a head delete): a failure is a ::warning::, never the run's.""" r = run(*args, check=False) @@ -122,17 +127,16 @@ def open_conflicts(lines, annotate="warning"): def resolve(slug, item): """A main SHA or PR number (`#` optional) -> full SHA on origin/main; None if it is not on main.""" - item = item.removeprefix("#") # the summary's paste command names PRs as #n - if re.fullmatch( - r"\d{1,6}", item - ): # PR numbers are short; SHAs given here are 7+ hex chars + item = item.removeprefix("#") # the summary names PRs as #n + if item.startswith("-"): + return None # never passed to git as an option + if re.fullmatch(r"\d{1,6}", item): # a PR number; SHAs are 7+ hex pr = json.loads(gh("api", f"repos/{slug}/pulls/{item}")) if not pr["merged"]: return None item = pr["merge_commit_sha"] - r = rm.git(REPO, "rev-parse", "-q", "--verify", f"{item}^{{commit}}", check=False) - sha = r.stdout.strip() - return sha if r.returncode == 0 and rm.is_ancestor(REPO, sha, "origin/main") else None + sha = rm.resolve(REPO, item) + return sha if sha and rm.is_ancestor(REPO, sha, MAIN) else None def main_order(repo, shas): @@ -146,19 +150,24 @@ def fix_class(slug, sha, token=None): """ try: out = gh("api", f"repos/{slug}/commits/{sha}/pulls", token=token) - except RuntimeError: # a read, and a rescan makes dozens: retry a transient error once ("unexpected end of JSON input") + except RuntimeError: # retry a transient read error once out = gh("api", f"repos/{slug}/commits/{sha}/pulls", token=token) prs = json.loads(out) if not prs: return None, "no PR", None pr = prs[0] return ( - rm.pr_fix_class(pr["title"], [label["name"] for label in pr["labels"]]), + rm.pr_fix_class(pr["title"], _labels(pr)), f"#{pr['number']}", pr["title"], ) +def _labels(pr): + """The label names of a gh PR.""" + return [label["name"] for label in pr["labels"]] + + def pr_body(sha, src, branch, head, files=()): """Body of the pick PR; files = conflicted paths (empty for a clean pick).""" origin = f"Backport of {src} (`{sha}`) to `{branch}`" @@ -187,16 +196,14 @@ def held_line(sha, cls, pr, branch, annotate="warning", slug=None): def held_warnings(awaiting): """One warning per stabilized branch (first-seen order) for awaiting [(sha, branch, class, PR ref)].""" - counts = {} - for _, b, *_ in awaiting: - counts[b] = counts.get(b, 0) + 1 + counts = collections.Counter(b for _, b, *_ in awaiting) return [ f"::warning::{b}: {n} held security/critical fix{'es' if n > 1 else ''}, Backport with confirm_stabilized" for b, n in counts.items() ] -STALE_DAYS = 5 # a pending item this many days old gets a stale mark +STALE_DAYS = 5 # pending this long gets ⚠ def link_pr(ref): @@ -208,14 +215,7 @@ def latest_tag(repo, branch): """(newest tag on the line, RC or stable, commits on the branch after it), or (None, None) before any. A stable tag ranks above the RC it was promoted from (same commit, same patch). """ - line = branch.split("/", 1)[1] - tags = [ - t - for t in rm.git( - repo, "tag", "--merged", f"origin/{branch}", "-l", f"{rs.PREFIX}v{line}.*" - ).split() - if (m := rs.VER_RE.fullmatch(t)) and f"{m[1]}.{m[2]}" == line - ] + tags = rs.line_tags(repo, branch.split("/", 1)[1], "--merged", f"origin/{branch}") if not tags: return None, None m = lambda t: rs.VER_RE.fullmatch(t) @@ -229,9 +229,7 @@ def next_step(conflicts, held_n, tag, unbuilt): return f"resolve {rm.pr_link(conflicts[0])}" if held_n: return f"RM: confirm {held_n} security fix" + ("es" if held_n > 1 else "") - driver = ( - "cloud " if rs.PREFIX else "" - ) # prefixed (OSS) lines are built and promoted by the cloud workflows + driver = "cloud " if rs.PREFIX else "" # OSS lines are built by the cloud workflows if unbuilt: return f"{driver}Take RC" if tag and "-rc." in tag: @@ -241,12 +239,12 @@ def next_step(conflicts, held_n, tag, unbuilt): def status_row(branch, state, tag, unbuilt, missing, conflicts, held_n): """One `| line | state | latest tag | missing | pending | next step |` row; conflicts: open conflict PR numbers.""" - built = ( - "none" - if tag is None - else f"{rm.tag_link(tag)} · " - + (f"+{unbuilt} not built" if unbuilt else "all built") - ) + if tag is None: + built = "none" + else: + built = f"{rm.tag_link(tag)} · " + ( + f"+{unbuilt} not built" if unbuilt else "all built" + ) pending = ", ".join( ([f"{len(conflicts)} conflict"] if conflicts else []) + ([f"{held_n} RM"] if held_n else []) @@ -310,10 +308,7 @@ def pending_block(conflicts, held_rows): + f" · merged {stale(d)} ago" for cls, pr, d, bs in groups.values() ] - commits = ",".join( - pr if re.fullmatch(r"#\d+", pr) else sha[:12] - for sha, (_, pr, _, _) in groups.items() - ) + commits = ",".join(_commit_arg(sha, pr) for sha, (_, pr, _, _) in groups.items()) branches = ",".join( dict.fromkeys(b for _, _, _, bs in groups.values() for b in bs) ) @@ -328,6 +323,11 @@ def pending_block(conflicts, held_rows): return out + ["
"] +def _commit_arg(sha, pr): + """A commit for Backport's commits input: its `#n` PR, else its sha.""" + return pr if re.fullmatch(r"#\d+", pr) else sha[:12] + + def push_summary(head, table, done, pending): """Push mode summary lines; a quiet push (nothing done, nothing pending) is the head line alone.""" if not done and not pending: @@ -372,7 +372,7 @@ def classes_from_listing(prs): if not pr.get("mergeCommit"): continue try: - cls = rm.pr_fix_class(pr["title"], [label["name"] for label in pr["labels"]]) + cls = rm.pr_fix_class(pr["title"], _labels(pr)) out[pr["mergeCommit"]["oid"]] = (cls, f"#{pr['number']}", pr["title"]) except ValueError as e: out[pr["mergeCommit"]["oid"]] = (UNCLASSIFIABLE, str(e), pr["title"]) @@ -384,7 +384,7 @@ def pr_classes(slug, since): listing per run, labels read now, so a label edited after the merge counts. A failed listing returns {}: held() then looks each commit up on its own. """ - # ponytail: the search API stops at 1000 results; a range with more merged PRs falls back per commit beyond them. + # Known limit: search stops at 1000 results; the rest are looked up per commit. try: out = gh( "pr", @@ -416,7 +416,7 @@ def held(slug, repo, branch, accept=HELD, cache=None, token=None): token: slug's, when slug is not this repo (the OSS line). """ remote = f"origin/{branch}" - base = rm.git(repo, "merge-base", "origin/main", remote) + base = rm.git(repo, "merge-base", MAIN, remote) # First parent, no merges: a merge-back's merge and the old release commits it brings in are not main's own. The # bot's direct commits (RC Helm bumps) are build-generated, as relmgr.classify_commit classes them. out = rm.git( @@ -439,19 +439,14 @@ def held(slug, repo, branch, accept=HELD, cache=None, token=None): if s in classes: continue looked += 1 + subject = lambda s=s: rm.git(repo, "log", "-1", "--format=%s", s) try: cls, pr, title = fix_class(slug, s, token=token) classes[s] = ( - (UNCLASSIFIABLE, pr, rm.git(repo, "log", "-1", "--format=%s", s)) - if pr == "no PR" - else (cls, pr, title) + (UNCLASSIFIABLE, pr, subject()) if pr == "no PR" else (cls, pr, title) ) except ValueError as e: # a title Lint PR should have stopped - classes[s] = ( - UNCLASSIFIABLE, - str(e), - rm.git(repo, "log", "-1", "--format=%s", s), - ) + classes[s] = (UNCLASSIFIABLE, str(e), subject()) print( f"held: {len(commits)} main commit(s) since the cut of {branch}, {looked} fix_class lookup(s)" ) @@ -509,18 +504,16 @@ def held_note(rows, branch, slug=None): """One summary line for held() rows of stabilized `branch`, or None. slug: the other repo the rows are from.""" if not rows: return None - commits = ",".join( - pr if re.fullmatch(r"#\d+", pr) else s[:12] for s, _, pr, _ in rows - ) - link = ( - ( - lambda pr: f"[{slug}{pr}](https://github.com/{slug}/pull/{pr[1:]})" - if pr.startswith("#") - else pr - ) - if slug - else link_pr - ) + commits = ",".join(_commit_arg(s, pr) for s, _, pr, _ in rows) + if slug: + + def link(pr): + if pr.startswith("#"): + return f"[{slug}{pr}](https://github.com/{slug}/pull/{pr[1:]})" + return pr + + else: + link = link_pr return ( f"⚠ Held for the RM on {f'{slug} ' if slug else ''}`{branch}` (stabilized; Promote blocks unless forced): " + ", ".join(f"{link(pr)} {cls}" for _, cls, pr, _ in rows) @@ -533,7 +526,7 @@ def rescan_targets(targets, listed): """({branch: accept} to rescan, listed branches that are no target): fix targets owe plain fixes too (OWED), the other security/critical targets (stabilized) only HELD. listed narrows, never adds. """ - scan = {b: OWED for b in targets["fix"]} + scan = dict.fromkeys(targets["fix"], OWED) for t in targets["security_critical"]: scan.setdefault(t["branch"], HELD) kept, others = only_listed(list(scan), listed) @@ -568,15 +561,13 @@ def pick_row(slug, sha, b, cls, pr, title): REPO, sha, f"origin/{b}" ) # advisory only: never skips the pick except Exception as e: - rel = f"unknown ({type(e).__name__}: {(str(e).splitlines() or [''])[0]})".replace( - "|", r"\|" - ) # a table cell + # A table cell. + rel = f"unknown ({type(e).__name__}: {(str(e).splitlines() or [''])[0]})" + rel = rel.replace("|", r"\|") print(f"::warning::{sha[:7]} relevance on {b}: {rel}") try: outcome, detail = pick(slug, sha, b, title, pr) - except ( - Exception - ) as e: # branches are independent: one failure never stalls the others + except Exception as e: traceback.print_exc() run("git", "cherry-pick", "--abort", check=False) outcome, detail = "failed", f"{type(e).__name__}: {e}" @@ -591,34 +582,52 @@ def pick_row(slug, sha, b, cls, pr, title): ) -def rescan(slug, targets, confirm, listed): - """Push mode: (rows, awaiting, {branch: missing}) over every target since its cut (see the module doc). - Known limit: no test drives rescan itself; rescan_plan and held are tested on their own. - """ - scan, others = rescan_targets(targets, listed) - rows = [] - for b in others: - print(f"::warning::{b} is not a target, skipped") - rows.append(("-" * 7, "-", "-", b, "skipped: not a target", "-")) - cuts = [rm.git(REPO, "merge-base", "origin/main", f"origin/{b}") for b in scan] +def _rescan_classes(slug, scan): + """pr_classes() since the oldest cut of scan's branches; {} with no branch to scan.""" + cuts = [rm.git(REPO, "merge-base", MAIN, f"origin/{b}") for b in scan] oldest = min( (rm.git(REPO, "log", "-1", "--format=%cI", c) for c in cuts), default=None ) + # A day's margin for clock and merge-time skew. since = ( (datetime.datetime.fromisoformat(oldest) - datetime.timedelta(days=1)).date() if oldest else None ) - classes = ( - pr_classes(slug, since.isoformat()) if since else {} - ) # a day's margin for clock and merge-time skew + return pr_classes(slug, since.isoformat()) if since else {} + + +def _rescan_missing(scan, rows, held_by_branch): + """{branch: missing} after the picks, each branch's tally printed.""" + missing = {} + for b in scan: + results = [r[4] for r in rows if r[3] == b] + picked = sum(res.startswith("merged") for res in results) + pending = sum( + res.startswith("pr ") or "pending" in res for res in results + ) # conflict PR, or merge held + missing[b] = len(held_by_branch[b]) - picked + print( + f"rescan {b}: {len(held_by_branch[b])} missing, {picked} picked, {pending} pending" + ) + return missing + + +def rescan(slug, targets, confirm, listed): + """Push mode: (rows, awaiting, {branch: missing}) over every target since its cut (see the module doc).""" + scan, others = rescan_targets(targets, listed) + rows = [] + for b in others: + print(f"::warning::{b} is not a target, skipped") + rows.append(("-" * 7, "-", "-", b, "skipped: not a target", "-")) + classes = _rescan_classes(slug, scan) listed_n = len(classes) held_by_branch = { b: held(slug, REPO, b, accept=accept, cache=classes) for b, accept in scan.items() } print(f"rescan: {listed_n} PRs listed, {len(classes) - listed_n} fallback lookup(s)") plan, awaiting, unclassifiable = rescan_plan(targets, held_by_branch, confirm) - # A human fixes the merged PR's title or adds a fix/ label (declines it when there is no PR); re-warned each push. + # Re-warned every push until the PR title or fix/ label is fixed (or the commit declined). for sha, b, pr, title in unclassifiable: print( f"::warning::{sha[:7]} ({pr}, {title}) cannot be classed for {b}: not picked" @@ -626,24 +635,11 @@ def rescan(slug, targets, confirm, listed): rows.append((sha, pr, UNCLASSIFIABLE, b, "skipped: unclassifiable", "-")) for b, sha, cls, pr, title in plan: rows.append(pick_row(slug, sha, b, cls, pr, title)) - missing = {} - for b in scan: - results = [r[4] for r in rows if r[3] == b] - picked = sum(res.startswith("merged") for res in results) - pending = sum( - res.startswith("pr ") or "pending" in res for res in results - ) # conflict PR, or merge held - missing[b] = len(held_by_branch[b]) - picked - print( - f"rescan {b}: {len(held_by_branch[b])} missing, {picked} picked, {pending} pending" - ) - return rows, awaiting, missing + return rows, awaiting, _rescan_missing(scan, rows, held_by_branch) PUSH_WAITS = (5, 20) # seconds before the 2nd and 3rd push attempt -PICK_TIMEOUT = ( - 30 * 60 -) # seconds: the pick job's timeout-minutes in backport.yaml; no run holds a head longer +PICK_TIMEOUT = 30 * 60 # = the pick job's timeout-minutes in backport.yaml def run_url(): @@ -680,17 +676,9 @@ def push_head(slug, sha, branch, head, src, sleep=time.sleep): ) if wait is not None: sleep(wait) - run( - "git", - "fetch", - "-q", - "origin", - f"+refs/heads/{branch}:refs/remotes/origin/{branch}", - check=False, - ) - if rm.is_converged( - REPO, sha, f"origin/{branch}", main="origin/main" - ): # the run that won the head landed it + fetch_branch(branch, check=False) + # The run that won the head landed it. + if rm.is_converged(REPO, sha, f"origin/{branch}", main=MAIN): return ( "skipped", f"already on branch (push of {head} rejected: another run landed the pick)", @@ -718,9 +706,8 @@ def push_head(slug, sha, branch, head, src, sleep=time.sleep): return "failed", f"push of {head} rejected {len(PUSH_WAITS) + 1} times: {first}" -CONFLICTS_RE = re.compile( - r"^Conflicts:\n\t", re.M -) # the block pick() adds to a conflict pick's message +# The block pick() adds to a conflict pick's message. +CONFLICTS_RE = re.compile(r"^Conflicts:\n\t", re.M) def fix_labels(slug, src): @@ -733,19 +720,31 @@ def fix_labels(slug, src): return [x for x in names if x.startswith("fix/")] +def _no_open_pr(branch, head): + """finish_open's outcome for a pick head with no open PR.""" + fetch_branch(head) + age = time.time() - int(rm.git(REPO, "log", "-1", "--format=%ct", f"origin/{head}")) + if age < PICK_TIMEOUT: + return ( + "skipped", + f"head {head} is {int(age // 60)} min old with no open PR: another run is picking it", + ) + # A head with no commit past the base: reopening its PR only parks it as a conflict PR with nothing to merge. + empty = rm.git(REPO, "rev-list", "--count", f"origin/{branch}..origin/{head}") == "0" + return ( + "failed", + f"head {head} exists with no open PR: delete it" + + ("" if empty else " or reopen its PR") + + ", then rerun", + ) + + def finish_open(slug, branch, head): - """The pick head already exists. An open unlabelled PR the bot opened, with one commit carrying the pick trailer - of the head's sha7, is a clean pick whose merge failed in an earlier run: squash it now, unless it is a conflict - pick that lost its label. A PR a person opened fails unmerged; a bot PR carrying any backport-* label waits for a - person; more than one commit or none, or one commit without that trailer, fails unmerged. A head with no open PR waits PICK_TIMEOUT for the run - that pushed it, then fails until a person deletes it or reopens its PR. - Known limits: the trailer is checked, not the commit's content, so a writer who force-pushes the bot's open pick - PR with one commit carrying that trailer gets it squashed (a writer can already run a workflow with the App's - token). A head whose PR was closed but whose decline record failed also reads as "no open PR": deleting that - head re-picks the declined fix; re-run the record job instead. A closed pick PR whose head has no commit past - the base also fails every run until the head is deleted (deliberately, see the module docstring). The age is - the head tip's commit time, so a head moved onto a fresh base tip reads as "another run is picking it" for up - to PICK_TIMEOUT. + """The pick head already exists. Squash an open bot PR whose single commit carries the head's pick trailer (a + clean pick whose merge failed earlier); any other PR fails or waits for a person. A head with no open PR waits + PICK_TIMEOUT, then fails until a person deletes the head or reopens its PR. + Known limits: only the trailer is checked, not the content; a head whose decline record failed also reads as + "no open PR" (re-run the record job, do not delete it); age is the head tip's commit time. """ pending = "skipped", f"conflict PR pending on `{head}`" prs = json.loads( @@ -764,34 +763,8 @@ def finish_open(slug, branch, head): "number,headRefOid,labels,author", ) ) - if ( - not prs - ): # a running run opens its PR after the push and deletes the head after the squash - run( - "git", - "fetch", - "-q", - "origin", - f"+refs/heads/{head}:refs/remotes/origin/{head}", - ) - age = time.time() - int( - rm.git(REPO, "log", "-1", "--format=%ct", f"origin/{head}") - ) - if age < PICK_TIMEOUT: - return ( - "skipped", - f"head {head} is {int(age // 60)} min old with no open PR: another run is picking it", - ) - # A head with no commit past the base: reopening its PR only parks it as a conflict PR with nothing to merge. - empty = ( - rm.git(REPO, "rev-list", "--count", f"origin/{branch}..origin/{head}") == "0" - ) - return ( - "failed", - f"head {head} exists with no open PR: delete it" - + ("" if empty else " or reopen its PR") - + ", then rerun", - ) + if not prs: # a live run opens its PR after pushing the head + return _no_open_pr(branch, head) if len(prs) != 1: return pending pr, remote = prs[0], f"origin/{branch}" @@ -802,7 +775,7 @@ def finish_open(slug, branch, head): return "failed", f"{why}: not merged" if any(x["name"].startswith("backport-") for x in pr["labels"]): return pending - run("git", "fetch", "-q", "origin", f"+refs/heads/{head}:refs/remotes/origin/{head}") + fetch_branch(head) # A person's commit on the bot's unlabelled PR (say a fixup after a failed squash): no conflict PR lists it. if (n := rm.git(REPO, "rev-list", "--count", f"{remote}..{pr['headRefOid']}")) != "1": return ( @@ -815,7 +788,7 @@ def finish_open(slug, branch, head): else "label it backport-conflict, then merge it, or close it to decline" ), ) - msg = rm.git(REPO, "log", "-1", "--format=%B", pr["headRefOid"]) + msg = rm.message(REPO, pr["headRefOid"]) if not rm.pick_trailer(head, msg): return ( "failed", @@ -831,6 +804,11 @@ def finish_open(slug, branch, head): return "failed", f"#{pr['number']} merge pending check" except RuntimeError as e: return "failed", f"#{pr['number']} merge retry: {str(e).splitlines()[0]}" + return _land(slug, branch, head, pr["number"], retried=True) + + +def _land(slug, branch, head, n, retried=False): + """After the squash of PR n: delete head, refetch branch (later picks stack on it), return the merged outcome.""" soft( f"delete of {head}", "gh", @@ -839,73 +817,63 @@ def finish_open(slug, branch, head): "DELETE", f"repos/{slug}/git/refs/heads/{head}", ) - run("git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}") - return ( - "merged", - f"{rm.git(REPO, 'rev-parse', '--short=7', remote)} (#{pr['number']}, retried)", + fetch_branch(branch) + short = rm.git(REPO, "rev-parse", "--short=7", f"origin/{branch}") + return "merged", f"{short} (#{n}{', retried' if retried else ''})" + + +def _open_pr(slug, branch, head, body, *opts): + """The URL of a new pick PR from head into branch; opts: its title and label flags.""" + return gh( + "pr", + "create", + "-R", + slug, + "--base", + branch, + "--head", + head, + *opts, + "--body", + body, + ) + + +def _land_clean(slug, sha, branch, title, src, head, labels): + """The outcome of a clean cherry-pick at HEAD: push it, open its PR and squash it.""" + pick_sha = rm.git(REPO, "rev-parse", "HEAD") + if failed := push_head(slug, sha, branch, head, src): + return failed + url = _open_pr( + slug, branch, head, pr_body(sha, src, branch, head), "--title", title, *labels ) + n = int(url.rsplit("/", 1)[1]) + # False: a required check is pending; PR and head stay for the next run. + if not bm.squash( + slug, {"number": n, "headRefOid": pick_sha}, rm.message(REPO, pick_sha) + ): + return "failed", f"{url} opened, merge pending check" + return _land(slug, branch, head, n) def pick(slug, sha, branch, title, src): """Returns (outcome, detail); outcome in merged | pr | skipped | failed. title: PR title; src: original PR ref.""" remote = f"origin/{branch}" # Fresh tip: an overlapping run may have merged this pick after this run fetched. - run("git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}") - if rm.is_converged(REPO, sha, remote, main="origin/main"): + fetch_branch(branch) + if rm.is_converged(REPO, sha, remote, main=MAIN): return "skipped", "already on branch" head = f"backport/{branch.split('/', 1)[1]}/{sha[:7]}" - if ( - run( - "git", "ls-remote", "--exit-code", "--heads", "origin", head, check=False - ).returncode - == 0 - ): # known limit: any other ls-remote failure (network, auth) reads as "no head", and the pick push then fails + # Known limit: an ls-remote error reads as "no head"; the push then fails. + ls = run("git", "ls-remote", "--exit-code", "--heads", "origin", head, check=False) + if ls.returncode == 0: return finish_open(slug, branch, head) labels = [x for name in fix_labels(slug, src) for x in ("--label", name)] run("git", "checkout", "-q", "--detach", remote) r = run("git", "cherry-pick", "-x", sha, check=False) if r.returncode == 0: - pick_sha = rm.git(REPO, "rev-parse", "HEAD") - if failed := push_head(slug, sha, branch, head, src): - return failed - url = gh( - "pr", - "create", - "-R", - slug, - "--base", - branch, - "--head", - head, - "--title", - title, - *labels, - "--body", - pr_body(sha, src, branch, head), - ) - n = int(url.rsplit("/", 1)[1]) - # backport_merge.squash: REST squash pinned to the head, with the pick's message. - # A required check GitHub still waits for -> False: PR and head stay, the next run reports it pending. - if not bm.squash( - slug, - {"number": n, "headRefOid": pick_sha}, - rm.git(REPO, "log", "-1", "--format=%B", pick_sha), - ): - return "failed", f"{url} opened, merge pending check" - soft( - f"delete of {head}", - "gh", - "api", - "-X", - "DELETE", - f"repos/{slug}/git/refs/heads/{head}", - ) - run( - "git", "fetch", "-q", "origin", f"+refs/heads/{branch}:refs/remotes/{remote}" - ) # later picks stack on it - return "merged", f"{rm.git(REPO, 'rev-parse', '--short=7', remote)} (#{n})" - # -z: a path with a space or a non-ASCII name comes back unquoted and in one piece (rm.git's strip can still - # drop a leading space on the first path; only the commit message and PR body read this list). + return _land_clean(slug, sha, branch, title, src, head, labels) + # -z keeps paths with spaces or non-ASCII intact (rm.git's strip may trim a leading space; display only). files = [ f for f in rm.git(REPO, "diff", "--name-only", "-z", "--diff-filter=U").split("\0") @@ -937,30 +905,23 @@ def pick(slug, sha, branch, title, src): run("git", "commit", "-q", "--no-verify", "-F", "-", input=msg) if failed := push_head(slug, sha, branch, head, src): return failed - url = gh( - "pr", - "create", - "-R", + url = _open_pr( slug, - "--base", branch, - "--head", head, + pr_body(sha, src, branch, head, files), "--label", "backport-conflict", *labels, "--title", title, - "--body", - pr_body(sha, src, branch, head, files), ) return "pr", url def push_title(slug, sha): """The push summary's head line: the pushed commit and its class.""" - # Known limit: a gh failure that survives fix_class's retry (RuntimeError) is not caught here and ends the run - # before its summary. + # Known limit: a gh error past fix_class's retry ends the run before the summary. try: cls, pr, _ = fix_class(slug, sha) except ValueError as e: # a title Lint PR should have stopped: the picks are done, only this line is left @@ -988,7 +949,7 @@ def main(): ) a = p.parse_args() slug = os.environ["GITHUB_REPOSITORY"] - with open(a.state) as f: + with open(rm.local_path(a.state)) as f: state = json.load(f) targets = state["backport_targets"] listed = [b.strip() for b in a.branches.split(",") if b.strip()] @@ -1001,9 +962,7 @@ def main(): print("\n".join(held_warnings(awaiting))) # a held pick is never silent now = datetime.datetime.now(datetime.UTC) - days = lambda iso: ( - now - datetime.datetime.fromisoformat(iso.replace("Z", "+00:00")) - ).days + days = lambda iso: (now - rm.ts(iso)).days conflicts = sorted( (p["number"], p["baseRefName"], days(p["createdAt"])) for p in conflict_prs(slug) ) @@ -1053,9 +1012,8 @@ def main(): def dispatch(slug, targets, commits, confirm, listed): """Dispatch mode: (rows, awaiting) for the given commits only.""" - items = [ - c for c in re.split(r"[,\s]+", commits) if c - ] # commas or spaces: the owed-picks line pastes as is + # Commas or spaces: the owed-picks line pastes as is. + items = [c for c in re.split(r"[,\s]+", commits) if c] shas = [resolve(slug, c) for c in items] if not items or None in shas: bad = [c for c, s in zip(items, shas, strict=False) if s is None] @@ -1064,45 +1022,55 @@ def dispatch(slug, targets, commits, confirm, listed): rows, awaiting = [], [] for sha in shas: - try: - cls, pr, title = fix_class(slug, sha) - except ValueError as e: # Lint PR should have stopped this PR - print(f"::error::{sha[:7]}: fix class: {e}") - rows.append((sha, "?", "error", "-", f"skipped: {e}", "-")) - continue - if cls is None: - route = ( - f"; label {pr} fix/needs-backport (or the fix label it needs) and rerun" - if pr.startswith("#") - else "" - ) - rows.append((sha, pr, "none", "-", f"skipped: not a fix{route}", "-")) - continue - if cls == "fix": - branches = targets["fix"] - else: - branches = [ - t["branch"] - for t in targets["security_critical"] - if confirm or not t["needs_rm_confirmation"] - ] - awaiting += [ - (sha, t["branch"], cls, pr) - for t in targets["security_critical"] - if t["needs_rm_confirmation"] - and not confirm - and (not listed or t["branch"] in listed) - ] - branches, others = only_listed(branches, listed) - others = [b for b in others if (sha, b, cls, pr) not in awaiting] - for b in others: - print(f"::warning::{sha[:7]} ({cls}, PR {pr}): {b} is not a target, skipped") - rows.append((sha, pr, cls, b, "skipped: not a target", "-")) - if not branches and not others: - rows.append((sha, pr, cls, "-", "skipped: no target branch", "-")) - rows += [pick_row(slug, sha, b, cls, pr, title) for b in branches] + rows += _dispatch_commit(slug, targets, sha, confirm, listed, awaiting) return rows, awaiting +def _dispatch_targets(targets, sha, cls, pr, confirm, listed): + """(branches to pick onto, awaiting rows held for the RM) for one commit of class cls.""" + if cls == "fix": + return targets["fix"], [] + branches = [ + t["branch"] + for t in targets["security_critical"] + if confirm or not t["needs_rm_confirmation"] + ] + return branches, [ + (sha, t["branch"], cls, pr) + for t in targets["security_critical"] + if t["needs_rm_confirmation"] + and not confirm + and (not listed or t["branch"] in listed) + ] + + +def _dispatch_commit(slug, targets, sha, confirm, listed, awaiting): + """The dispatch rows for one commit; its held picks are appended to awaiting.""" + try: + cls, pr, title = fix_class(slug, sha) + except ValueError as e: # Lint PR should have stopped this PR + print(f"::error::{sha[:7]}: fix class: {e}") + return [(sha, "?", "error", "-", f"skipped: {e}", "-")] + if cls is None: + route = ( + f"; label {pr} fix/needs-backport (or the fix label it needs) and rerun" + if pr.startswith("#") + else "" + ) + return [(sha, pr, "none", "-", f"skipped: not a fix{route}", "-")] + branches, held_picks = _dispatch_targets(targets, sha, cls, pr, confirm, listed) + awaiting += held_picks + rows = [] + branches, others = only_listed(branches, listed) + others = [b for b in others if (sha, b, cls, pr) not in awaiting] + for b in others: + print(f"::warning::{sha[:7]} ({cls}, PR {pr}): {b} is not a target, skipped") + rows.append((sha, pr, cls, b, "skipped: not a target", "-")) + if not branches and not others: + rows.append((sha, pr, cls, "-", "skipped: no target branch", "-")) + rows += [pick_row(slug, sha, b, cls, pr, title) for b in branches] + return rows + + if __name__ == "__main__": sys.exit(main()) diff --git a/.github/release/backport_merge.py b/.github/release/backport_merge.py index 56b50f8b7d..5ae8cd51ab 100644 --- a/.github/release/backport_merge.py +++ b/.github/release/backport_merge.py @@ -9,17 +9,11 @@ state is read again here, so a re-run or a repeated event acts on what the PR is now. Labels on backport PRs are the bot's: each case but a head with no pick sets exactly one `backport-*` label and removes the others. - merged: label `backport-merged`; delete the head if it is still there. -- closed unmerged: a decline. The reason is the closer's newest comment made in the 2 minutes up to the close (the - close comment), first line, a leading `Declined:` dropped; with none, `Closed by @ without a reason (PR - #)`. Push an empty commit onto the base with the pick's message, the trailer and a final `Declined: ` - line; label `backport-declined`; comment `Recorded as a decline on (). To undo: ...` once per close. - The head is kept so the PR can be reopened. A pick or decline already on the base (newest first) -> no commit, - label and comment still fixed. The decline's message is the main commit's own, so a head edited by hand never - names other fixes: the head's first commit must carry exactly one pick trailer, naming the main commit of the - head's sha7. Otherwise -> no decline commit: the bot comments and the run fails. A head with no commit past the - base (moved onto it) -> nothing declined, no label and no delete: the bot comments once. While the head is still - this PR's, the comment says to delete it by hand, or to keep it until a hand-made decline is on the base; once - the head is gone or holds another pick, it says only that nothing was declined. +- closed unmerged: a decline. Reason: the closer's last comment within 2 min of the close (first line, Declined: + dropped), else "Closed by @ without a reason (PR #)". Push an empty commit on the base with the pick's message, + trailer and Declined line; label backport-declined; comment once. Head kept. Already on the base: no commit. Head + without exactly one matching trailer: comment, fail. Head with no commit past the base: comment once, nothing + declined. - open (reopened): label `backport-conflict`, nothing else. Merging it then lands the pick after the decline, and the newest trailer on the line wins (relmgr.trailer_states): the fix counts as shipped again. Run from a full clone of main, GH_TOKEN = the release App's installation token, RELEASE_BOT_APP_SLUG = its slug. @@ -42,8 +36,10 @@ def is_backport(head, base): """Is head a pick head for base (backport/X.Y/ into release/X.Y)?""" - m = rm.BACKPORT_HEAD_RE.fullmatch(head) - return bool(m) and base == f"release/{m[1]}" + return ( + rm.is_backport(head) + and base == f"release/{rm.BACKPORT_HEAD_RE.fullmatch(head)[1]}" + ) def gh(*args, token=None): @@ -60,13 +56,11 @@ def decline_reason(comments, sender, closed_at): """First line of the closer's newest comment in the REASON_WINDOW up to closed_at, a leading `Declined:` dropped; None if there is none or it is empty. """ - at = lambda t: datetime.datetime.fromisoformat(t.replace("Z", "+00:00")) - end = at(closed_at) + end = rm.ts(closed_at) for c in sorted(comments, key=lambda c: c["created_at"], reverse=True): - if c["user"] == sender and end - REASON_WINDOW <= at(c["created_at"]) <= end: + if c["user"] == sender and end - REASON_WINDOW <= rm.ts(c["created_at"]) <= end: line = (c["body"].strip().splitlines() or [""])[0] - # A pick trailer in the reason would mark that other commit declined in the decline commit's message. - # Strip until none is left: a trailer nested in another reassembles after one pass. + # Strip pick trailers (repeatedly: nested ones reassemble) so the reason cannot decline another commit. while rm.PICK_RE.search(line): line = rm.PICK_RE.sub("", line) return line.removeprefix("Declined:").strip() or None @@ -87,7 +81,7 @@ def head_messages(pr): f"+refs/pull/{n}/head:refs/backport/{n}", ) return [ - rm.git(".", "log", "-1", "--format=%B", c) + rm.message(".", c) for c in rm.git(".", "rev-list", "--reverse", f"origin/{base}..{sha}").split() ] @@ -100,18 +94,18 @@ def first_pick(pr): msgs = head_messages(pr) if not msgs: return None, [] - o = rm.pick_trailer(pr["headRefName"], msgs[0] if msgs else "") + o = rm.pick_trailer(pr["headRefName"], msgs[0]) if not o or not rm.on_main(".", o, "origin/main"): return "", [] full = rm.git(".", "rev-parse", f"{o}^{{commit}}") return ( - f"{rm.git('.', 'log', '-1', '--format=%B', full).strip()}\n\n(cherry picked from commit {full})", + f"{rm.message('.', full).strip()}\n\n(cherry picked from commit {full})", [full], ) def landed(originals, commits): - """First (sha, message) in commits whose pick trailer names one of originals; None if the pick is not there.""" + """Sha of the first (sha, message) whose pick trailer names one of originals, else None.""" for sha, msg in commits: if set(rm.PICK_RE.findall(msg)) & set(originals): return sha @@ -160,9 +154,8 @@ def squash(slug, pr, msg, sleep=time.sleep): ) return True except RuntimeError as e: - # A required check GitHub still waits for: the caller reports it. Accepted: this matches GitHub's - # "Required status check ... is expected/in progress" wording, which GitHub does not document as stable. - # Known limit: a 405 naming a failed status check matches too and reads as pending. + # Required check pending: the caller reports it. Accepted: matches GitHub's undocumented "status check" + # wording, so a failed check also reads as pending. if "status check" in str(e).lower(): print(f"#{pr['number']}: {str(e).splitlines()[0]}") return False @@ -245,9 +238,8 @@ def decline(slug, pr, reason): raise RuntimeError( f"#{n}: no single pick trailer for {pr['headRefName']}, decline not recorded" ) - if done := landed( - originals, base_picks(base, originals) - ): # re-run, or a close after a reopen: nothing to write + # A re-run, or a close after a reopen: nothing to write. + if done := landed(originals, base_picks(base, originals)): return done, False # Empty commit = the base's own tree. REST, so author and committer are the bot token's user, as on merge. tip, tree = base_tip(base) @@ -278,16 +270,11 @@ def decline(slug, pr, reason): return new, True -def close(slug, n, sender): - return record(slug, n, sender)["result"] - - def bot_refusal(pr): """Why `pr` (gh's number and author) is not a PR the bot opened, or None. - A head name and trailer are strings any writer can push; only a PR the bot opened is its pick. Backport opens - pick PRs with the release App's installation token, so they are authored by app/. That token cannot - read `gh api user`, so the slug comes from the workflow. + Head names and trailers are forgeable; only a PR authored by app/ is the bot's (the App + token cannot read gh api user). """ if not (app := os.environ.get("RELEASE_BOT_APP_SLUG")): return f"#{pr['number']}: RELEASE_BOT_APP_SLUG is unset (the release App's slug)" @@ -299,6 +286,39 @@ def bot_refusal(pr): return None +def _no_pick(slug, n, pr, cs): + """What record did for a closed PR whose head has no pick to decline; cs: its comments.""" + base, head = pr["baseRefName"], pr["headRefName"] + kept = head_ref(slug, head) == pr["headRefOid"] + if not any( + c["body"].startswith(NO_PICK) and c["created_at"] >= pr["closedAt"] for c in cs + ): + gh( + "pr", + "comment", + str(n), + "-R", + slug, + "--body", + f"{NO_PICK} {base}, so nothing was declined. " + + ( + f"Until `{head}` is deleted, Backport fails on it. To have the fix picked again, delete " + f"`{head}` by hand. To decline the fix instead, do not delete the head yet: ask the release " + f"manager for a hand-made empty commit on {base} with the pick's message, its `(cherry picked " + "from commit )` line and a final `Declined: ` line. Once it is on " + f"{base}, Backport stops failing and the head can be deleted." + if kept + else f"`{head}` no longer holds this PR's head, so there is nothing to delete here." + ), + ) + return { + "result": "skipped: no pick" + ("" if kept else ", head gone"), + "base": base, + "sha": None, + "reason": None, + } + + def record(slug, n, sender): """What close did: {result, base, sha, reason}; sha is the squash, decline or existing pick, reason the decline's.""" pr = json.loads( @@ -340,35 +360,7 @@ def record(slug, n, sender): # No pick on the head: nothing to decline. Removing the head is manual: until a person deletes it, every # Backport run fails on it. The head may already be gone (deleting it closes the PR) or hold a later run's pick. if not sha: - kept = head_ref(slug, head) == pr["headRefOid"] - if not any( - c["body"].startswith(NO_PICK) and c["created_at"] >= pr["closedAt"] - for c in cs - ): - gh( - "pr", - "comment", - str(n), - "-R", - slug, - "--body", - f"{NO_PICK} {base}, so nothing was declined. " - + ( - f"Until `{head}` is deleted, Backport fails on it. To have the fix picked again, delete " - f"`{head}` by hand. To decline the fix instead, do not delete the head yet: ask the release " - f"manager for a hand-made empty commit on {base} with the pick's message, its `(cherry picked " - "from commit )` line and a final `Declined: ` line. Once it is on " - f"{base}, Backport stops failing and the head can be deleted." - if kept - else f"`{head}` no longer holds this PR's head, so there is nothing to delete here." - ), - ) - return { - "result": "skipped: no pick" + ("" if kept else ", head gone"), - "base": base, - "sha": None, - "reason": None, - } + return _no_pick(slug, n, pr, cs) set_label(slug, pr, DECLINED) if not any( c["body"].startswith("Recorded as a decline") diff --git a/.github/release/convgate.py b/.github/release/convgate.py index d960e0fef3..21beb3515c 100644 --- a/.github/release/convgate.py +++ b/.github/release/convgate.py @@ -4,31 +4,28 @@ Usage: convgate.py --branch release/X.Y --target T [--promote-ref vX.Y.Z-rc.N] [--state state.json] [--repo DIR] [--oss-repo DIR] [--annotate warning|error] [--out FILE] -Wraps relmgr.convergence_report unchanged and adds: +Wraps relmgr.convergence_report and adds: - scope: relstate `supported[T]` minus the branch. Retired and not-enabled branches are out by construction. - direction: an older branch's picks are checked against the newer branch, whichever side is being promoted. - reference: a newer branch with a promoted stable tag for T is scanned at that tag (what ships next), not at its tip, even before it is GA (on-prem: promoted, chart not published yet). The branch being promoted is scanned at - --promote-ref where it is the newer side. As the older side it is scanned at its tip, deliberately: a fix picked - onto it after the RC still blocks the promote until the newer line has it. + --promote-ref where it is the newer side. As the older side it is scanned at its tip, so a fix picked after the + RC still blocks. - declines shown: an older line's pick the newer line declined prints as DECLINED, with the reason and the decline commit (relmgr.decline_records). It counts as converged. A decline on the older line owes the newer line nothing. -- release-branch-only window: an RBO-ABSENT row is kept only for a commit dated after the newer line's latest GA - promoted stable tag for T (its cut, the merge-base with main, when it has none yet) that touches a path outside .github/. - The RM saw older ones at the last promote, and workflow-only commits carry nothing a product line needs. A dropped - commit leaves the counts and the report. +- release-branch-only window: RBO-ABSENT is kept only for commits after the newer line's latest promoted tag (else + its cut) that touch paths outside .github/. - OSS pins (a repo whose lines pin an OSS tag, relstate PINS): the same scan in a clone of the OSS repo, the older line's pinned OSS tag into the newer line's, so an upgrade never loses an OSS fix. Each line is read at the same reference as above. A MISSING row names where to act: the OSS line (OSS Backport) or a new RC of the newer line. Exit 2 iff any MISSING or an older line is not GA yet (promote order), 0 otherwise, 1 on error (never a silent pass). Take RC build warns on 2; Promote blocks. -Each MISSING row names what blocks the promote and where to act (the open Backport conflict PR, if any, or -the Backport run whose push was rejected). checks() turns a gate result into the checks table both summaries lead with, and the full rows folded below it. """ import argparse import datetime as dt +import functools import json import os import re @@ -41,6 +38,7 @@ import relstate as rs STATUSES = ("CONVERGED", "DECLINED", "MISSING", "UNCLASSIFIABLE", "RBO-ABSENT") +CLOSE_HOWTO = "resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." def conflict_pr(head): @@ -82,7 +80,7 @@ def push_failure(number, branch): capture_output=True, text=True, ) - if r.returncode: # never fall back to "no PR found" when the lookup itself failed + if r.returncode: # never read a failed lookup as "no failure comment" raise RuntimeError(f"gh api issues/{number}/comments: {r.stderr.strip()}") m = re.search(r"\(run (\S+)\)", (r.stdout.strip().splitlines() or [""])[-1]) return m and m[1] @@ -106,15 +104,12 @@ def missing_message( failure(pr number, newer): the run URL of Backport's push-failure comment on the source PR, or None. """ o7 = orig[:7] - subject = rm.git( - repo, "log", "-1", "--format=%s", orig, check=False - ).stdout.strip() # orig may not be on main - m = re.search(r"\(#(\d+)\)$", subject) - fix = f"fix {o7}" + (f" (PR #{m[1]})" if m else "") + # orig may not be on main. + n = rm.subject_pr(repo, orig, check=False) + fix = f"fix {o7}" + (f" (PR #{n})" if n is not None else "") head = blocked(branch, target, annotate) - if ( - unshipped and newer == branch and rs.VER_RE.fullmatch(unshipped)[4] - ): # an RC of this line, cut before the fix + # An RC of this line, cut before the fix. + if unshipped and newer == branch and rs.VER_RE.fullmatch(unshipped)[4]: return f"{head}: {fix} is on {newer} but not in RC {unshipped}: take a new RC build first." if unshipped: # nothing to pick or decline: the fix only waits for newer's next patch return f"{head}: {fix} is on {newer} but not in its latest stable {unshipped}: promote a {newer} patch first." @@ -123,70 +118,75 @@ def missing_message( f"{head}: {fix} is not on newer {newer} yet; an older line never ships a fix its newer line lacks, " "even one picked after the RC being promoted." ) - act = ( - f"Conflict PR #{pr[0]} on {newer}: resolve, get one approval and merge, or close the PR to decline (the close comment is the reason)." - if pr - else None - ) + act = f"Conflict PR #{pr[0]} on {newer}: {CLOSE_HOWTO}" if pr else None else: msg = f"{head}: {fix} is on older {older} but not on this line." - act = ( - f"Conflict PR #{pr[0]} is on this branch: resolve, get one approval and merge, or close the PR to " - "decline (the close comment is the reason)." - if pr - else None - ) - if ( - not act and m and failure and (url := failure(int(m[1]), newer)) - ): # backport.push_head wrote it + act = f"Conflict PR #{pr[0]} is on this branch: {CLOSE_HOWTO}" if pr else None + # The failure comment comes from backport.push_head. + if not act and n is not None and failure and (url := failure(n, newer)): act = f"Backport for {o7} to {newer} failed: push rejected (run {url}). Rerun Backport for {o7} on {newer}." return f"{msg} {act or f'No Backport PR found for it: run Backport for {o7}, or decline it on {newer}.'}" -def oss_pins(repo, oss_repo, branch, target, scope, ref, rows, annotate): - """(missing, report lines, per-line results): each older line's pinned OSS tag scanned into the newer line's.""" - main, oss_refs = "refs/remotes/origin/main", rs._branches(oss_repo) - pins = {n: oss.pin(repo, ref(n)) for n in [branch, *scope]} +def _oss_pins_of(repo, oss_repo, names, ref): + """{line: its pinned OSS tag}, each tag checked present in oss_repo.""" + pins = {n: oss.pin(repo, ref(n)) for n in names} for p in set(pins.values()): if not rs._tag_commit(oss_repo, p): raise RuntimeError(f"no OSS tag {p} in {oss_repo}") + return pins + + +def _oss_found(oss_repo, main, oss_refs, p_old, p_new, line_new): + """The CONVERGED and MISSING rows of the OSS picks between two pins.""" + if p_old == p_new: + return [] + return [ + r + for r in rm.convergence_report( + oss_repo, + f"refs/tags/{p_old}", + f"refs/tags/{p_new}", + main=main, + newer_branch=oss_refs.get(line_new), + ) + # Accepted: a pin counts only CONVERGED and MISSING rows. + if r[1] in ("CONVERGED", "MISSING") + ] + + +def _oss_act(oss_repo, main, oss_refs, o, newer, line_new): + """(fix, act) for the OSS original o missing from newer's pin.""" + n = rm.subject_pr(oss_repo, o) + fix = f"OSS fix {o[:7]}" + (f" (OSS PR #{n})" if n is not None else "") + on_line = line_new in oss_refs and rm.is_converged( + oss_repo, o, oss_refs[line_new], main + ) + act = ( + f"take a new RC of {newer} to pin OSS {line_new}'s tip" + if on_line + else f"pick it onto OSS {line_new} (OSS Backport), then take a new RC of {newer}" + ) + return fix, act + + +def oss_pins(repo, oss_repo, branch, target, scope, ref, rows, annotate): + """(missing, report lines, per-line results): each older line's pinned OSS tag scanned into the newer line's.""" + main, oss_refs = "refs/remotes/origin/main", rs._branches(oss_repo) + pins = _oss_pins_of(repo, oss_repo, [branch, *scope], ref) head = blocked(branch, target, annotate) missing, lines, out = 0, [], [f"### OSS pins: `{branch}` pins `{pins[branch]}`", ""] for s in scope: - older, newer = sorted((s, branch), key=lambda n: rs._key(n.split("/", 1)[1])) + older, newer = rs.older_newer(s, branch) p_old, p_new, line_new = pins[older], pins[newer], oss.line_of(pins[newer]) - found = ( - [] - if p_old == p_new - else [ - r - for r in rm.convergence_report( - oss_repo, - f"refs/tags/{p_old}", - f"refs/tags/{p_new}", - main=main, - newer_branch=oss_refs.get(line_new), - ) - # Accepted: only CONVERGED and MISSING count for a pin; DECLINED, UNCLASSIFIABLE and RBO-ABSENT - # OSS picks are left out of the counts and the detail. - if r[1] in ("CONVERGED", "MISSING") - ] - ) + found = _oss_found(oss_repo, main, oss_refs, p_old, p_new, line_new) n_miss = sum(st == "MISSING" for _, st in found) missing += n_miss summary = f"{len(found)} OSS picks, {len(found) - n_miss} in {p_new}" + ( f", {n_miss} missing" if n_miss else "" ) - line = { - "line": s, - "side": "older" if s == older else "newer", - "picks": len(found), - "summary": summary, - "todo": [], - "rbo": [], - "state": rows[s]["targets"].get(target, {}).get("state"), - "oss": True, - } + state_s = rows[s]["targets"].get(target, {}).get("state") + line = _line_result(s, older, len(found), summary, state_s, oss=True) lines.append(line) out.append( f"**{s}** ({line['side']}; `{older}` pins `{p_old}`, `{newer}` pins `{p_new}`): {summary}" @@ -195,18 +195,7 @@ def oss_pins(repo, oss_repo, branch, target, scope, ref, rows, annotate): if st != "MISSING": continue for o in rm.picked_from(oss_repo, c): - m = re.search( - r"\(#(\d+)\)$", rm.git(oss_repo, "log", "-1", "--format=%s", o) - ) - fix = f"OSS fix {o[:7]}" + (f" (OSS PR #{m[1]})" if m else "") - on_line = line_new in oss_refs and rm.is_converged( - oss_repo, o, oss_refs[line_new], main - ) - act = ( - f"take a new RC of {newer} to pin OSS {line_new}'s tip" - if on_line - else f"pick it onto OSS {line_new} (OSS Backport), then take a new RC of {newer}" - ) + fix, act = _oss_act(oss_repo, main, oss_refs, o, newer, line_new) msg = f"{head}: {fix} is in {older}'s OSS pin {p_old} but not in {newer}'s pin {p_new}: {act}." print(f"::{annotate}::{msg}") out.append(f"- `{c[:12]}` MISSING original {o[:12]}") @@ -216,6 +205,180 @@ def oss_pins(repo, oss_repo, branch, target, scope, ref, rows, annotate): return missing, out, lines +def _line_result(s, older, picks, summary, state, oss=False): + """The per-line result checks() reads, for line s scanned against branch.""" + return { + "line": s, + "side": "older" if s == older else "newer", + "picks": picks, + "summary": summary, + "todo": [], + "rbo": [], + "state": state, + **({"oss": True} if oss else {}), + } + + +def _tag_ref(repo, tag): + """refs/tags/, after checking the tag exists.""" + if not rs._tag_commit(repo, tag): + raise RuntimeError(f"no git tag {tag}") + return f"refs/tags/{tag}" + + +def _ref(repo, refs, rows, branch, target, promote_ref, name): + """The ref gate() scans line `name` at.""" + if name == branch and promote_ref: + return _tag_ref(repo, promote_ref) + p = rows[name]["targets"].get(target, {}).get("promoted") + return _tag_ref(repo, p["tag"]) if p else refs[name] + + +def _since(repo, main, newer_tip, promoted): + """Unix time the release-branch-only window opens: the promoted stable tag, else the cut.""" + if promoted: + return int( + rm.git( + repo, + "for-each-ref", + "--format=%(creatordate:unix)", + f"refs/tags/{promoted['tag']}", + ) + ) + return int( + rm.git( + repo, + "log", + "-1", + "--format=%ct", + rm.git(repo, "merge-base", main, newer_tip), + ) + ) + + +def _in_window(repo, c, since): + """Whether release-branch-only commit c is dated after since and touches a path outside .github/.""" + return int(rm.git(repo, "log", "-1", "--format=%ct", c)) > since and any( + not f.startswith(".github/") + for f in rm.git(repo, "show", "--format=", "--name-only", c).split() + ) + + +def _scan(repo, main, refs, ref, older, newer, since): + """(rows, picks): each (commit, status, originals, decline) of older scanned into newer.""" + rows_out, n_picks = [], 0 + recs = rm.decline_records(repo, refs[newer], main) # declines on newer's tip + rec = lambda o, recs=recs: next( + (v for k, v in recs.items() if rm.same_sha(k, o)), None + ) + for c, status in rm.convergence_report( + repo, refs[older], ref(newer), main=main, newer_branch=refs[newer] + ): + if status == "RBO-ABSENT" and not _in_window(repo, c, since): + continue # outside the window, or nothing outside .github/ (an empty commit included) + origs = rm.picked_from(repo, c) + n_picks += bool(origs) + decline = ( + rec(origs[0]) + if status == "CONVERGED" and origs and all(rec(o) for o in origs) + else None + ) + if decline: + status = "DECLINED" + rows_out.append((c, status, origs, decline)) + return rows_out, n_picks + + +def _counts(rows_out): + """{status: number of rows_out with it}.""" + return {st: sum(r[1] == st for r in rows_out) for st in STATUSES} + + +def _shipped(promoted, newer, branch, promote_ref): + """The summary's name for newer's reference: what newer is scanned at.""" + return ( + "the RC" if not promoted or newer == branch and promote_ref else promoted["tag"] + ) + + +def _summary(n, n_picks, shipped, newer): + """A line's counts, from the per-status counts n.""" + summary = f"{n_picks} picks, {n['CONVERGED']} in {shipped}" + "".join( + f", {n[k]} {word}" + for k, word in ( + ("DECLINED", f"declined on {newer}"), + ("MISSING", "missing"), + ("UNCLASSIFIABLE", "unclassifiable"), + ) + if n[k] + ) + if n["RBO-ABSENT"]: + summary += f" · {n['RBO-ABSENT']} release-branch-only to check" + return summary + + +def _rbo_what(repo, c): + """The ticket(s), title and link of release-branch-only commit c.""" + body = rm.git(repo, "log", "-1", "--format=%s%n%n%b", c) + tickets, title = ( + list(dict.fromkeys(rm.RBO_RE.findall(body))), + body.splitlines()[0], + ) + # Skip the ticket when the title starts with it. + lead = "" if title.split(" ", 1)[0] in tickets else ", ".join(tickets) + " " + return f"{lead}{title} ({rm.commit_link(c)})" + + +def _unshipped(repo, main, refs, ref, newer, o, pr): + """The tag newer is scanned at when the fix o, with no conflict PR, is on newer's tip but not in it, else None.""" + shipped = ref(newer) + return ( + shipped.removeprefix("refs/tags/") + if not pr + and shipped != refs[newer] + and rm.is_converged(repo, o, refs[newer], main) + else None + ) + + +def _todo(pr, newer, branch, msg): + """The do-this item for one MISSING original.""" + if not pr: + return msg.split(": ", 1)[1] + then = ", then Take RC again" if newer == branch else f" on `{newer}`" + return f"merge or close conflict PR {rm.pr_link(pr[0])}" + then + + +def _row_out( + repo, main, refs, ref, lookup, message, annotate, branch, older, newer, row, out, line +): + """Append one scanned row's detail to out and its do-this or to-check item to line.""" + c, status, origs, decline = row + if status == "DECLINED": + d, why = decline + out.append( + f"- `{c[:12]}` DECLINED on {newer} original {', '.join(o[:12] for o in origs)}: {why} ({d[:7]})" + ) + # RBO-ABSENT has no original: name the ticket and commit. + elif status == "RBO-ABSENT": + what = _rbo_what(repo, c) + out.append(f"- `{c[:12]}` RBO-ABSENT {what}") + line["rbo"].append(f"on `{older}`: {what}, not on `{newer}`") + elif status != "CONVERGED": + out.append( + f"- `{c[:12]}` {status} original {', '.join(o[:12] for o in origs) or '-'}" + ) + if status == "MISSING": + # the developer's next step, on the run page and in the summary + for o in origs: + pr = lookup(f"backport/{newer.split('/', 1)[1]}/{o[:7]}") + unshipped = _unshipped(repo, main, refs, ref, newer, o, pr) + msg = message(o, pr, unshipped=unshipped) + print(f"::{annotate}::{msg}") + out.append(f" - {msg}") + line["todo"].append(_todo(pr, newer, branch, msg)) + + def gate( repo, branch, @@ -234,163 +397,56 @@ def gate( if branch not in refs or branch not in rows: raise RuntimeError(f"{branch} not found") - def tag_ref(tag): - if not rs._tag_commit(repo, tag): - raise RuntimeError(f"no git tag {tag}") - return f"refs/tags/{tag}" - - def ref(name): # a promoted branch ships its latest stable tag for T, not its tip - if name == branch and promote_ref: - return tag_ref(promote_ref) - p = rows[name]["targets"].get(target, {}).get("promoted") - return tag_ref(p["tag"]) if p else refs[name] - + # a promoted branch ships its latest stable tag for T, not its tip + ref = functools.partial(_ref, repo, refs, rows, branch, target, promote_ref) scope = [s for s in state["supported"][target] if s != branch] out = [f"### Convergence gate: `{branch}` for {target} at `{ref(branch)}`", ""] if not scope: out.append("scanned branches: none") missing, lines = 0, [] for s in scope: - older, newer = sorted((s, branch), key=lambda n: rs._key(n.split("/", 1)[1])) + older, newer = rs.older_newer(s, branch) promoted = rows[newer]["targets"].get(target, {}).get("promoted") - since = int( - rm.git( - repo, - "for-each-ref", - "--format=%(creatordate:unix)", - f"refs/tags/{promoted['tag']}", - ) - if promoted - else rm.git( - repo, - "log", - "-1", - "--format=%ct", - rm.git(repo, "merge-base", main, refs[newer]), - ) - ) - rows_out, n_picks = [], 0 - recs = rm.decline_records(repo, refs[newer], main) # declines on newer's tip - rec = lambda o, recs=recs: next( - (v for k, v in recs.items() if k.startswith(o) or o.startswith(k)), None - ) # trailers may be abbreviated - for c, status in rm.convergence_report( - repo, refs[older], ref(newer), main=main, newer_branch=refs[newer] - ): - if status == "RBO-ABSENT" and ( - int(rm.git(repo, "log", "-1", "--format=%ct", c)) <= since - or not any( - not f.startswith(".github/") - for f in rm.git(repo, "show", "--format=", "--name-only", c).split() - ) - ): - continue # outside the window, or nothing outside .github/ (an empty commit included) - origs = rm.picked_from(repo, c) - n_picks += bool(origs) - decline = ( - rec(origs[0]) - if status == "CONVERGED" and origs and all(rec(o) for o in origs) - else None - ) - if decline: - status = "DECLINED" - rows_out.append((c, status, origs, decline)) - n = {st: sum(r[1] == st for r in rows_out) for st in STATUSES} + since = _since(repo, main, refs[newer], promoted) + rows_out, n_picks = _scan(repo, main, refs, ref, older, newer, since) + n = _counts(rows_out) missing += n["MISSING"] state_s = rows[s]["targets"].get(target, {}).get("state") - shipped = ( - "the RC" - if not promoted or newer == branch and promote_ref - else promoted["tag"] - ) # what newer is scanned at - summary = f"{n_picks} picks, {n['CONVERGED']} in {shipped}" + "".join( - f", {n[k]} {word}" - for k, word in ( - ("DECLINED", f"declined on {newer}"), - ("MISSING", "missing"), - ("UNCLASSIFIABLE", "unclassifiable"), - ) - if n[k] + summary = _summary( + n, n_picks, _shipped(promoted, newer, branch, promote_ref), newer ) - if n["RBO-ABSENT"]: - summary += f" · {n['RBO-ABSENT']} release-branch-only to check" - line = { - "line": s, - "side": "older" if s == older else "newer", - "picks": n_picks, - "summary": summary, - "todo": [], - "rbo": [], - "state": state_s, - } + line = _line_result(s, older, n_picks, summary, state_s) lines.append(line) out.append( f"**{s}** ({line['side']}, {state_s or 'not enabled'} for {target}; " f"`{refs[older]}` into `{ref(newer)}`): {summary}" ) - for c, status, origs, decline in rows_out: - if status == "DECLINED": - d, why = decline - out.append( - f"- `{c[:12]}` DECLINED on {newer} original {', '.join(o[:12] for o in origs)}: {why} ({d[:7]})" - ) - elif ( - status == "RBO-ABSENT" - ): # no original to name: the ticket and the commit instead - body = rm.git(repo, "log", "-1", "--format=%s%n%n%b", c) - tickets, title = ( - list(dict.fromkeys(rm.RBO_RE.findall(body))), - body.splitlines()[0], - ) - lead = ( - "" if title.split(" ", 1)[0] in tickets else ", ".join(tickets) + " " - ) # the title may name it - what = f"{lead}{title} ({rm.commit_link(c)})" - out.append(f"- `{c[:12]}` RBO-ABSENT {what}") - line["rbo"].append(f"on `{older}`: {what}, not on `{newer}`") - elif status != "CONVERGED": - out.append( - f"- `{c[:12]}` {status} original {', '.join(o[:12] for o in origs) or '-'}" - ) - if status == "MISSING": - for o in ( - origs - ): # the developer's next step, on the run page and in the summary - pr = lookup(f"backport/{newer.split('/', 1)[1]}/{o[:7]}") - shipped = ref( - newer - ) # newer's promoted stable tag when it has one, else its tip - unshipped = ( - shipped.removeprefix("refs/tags/") - if not pr - and shipped != refs[newer] - and rm.is_converged(repo, o, refs[newer], main) - else None - ) - msg = missing_message( - repo, - branch, - target, - older, - newer, - o, - pr, - annotate, - unshipped, - failure, - ) - print(f"::{annotate}::{msg}") - out.append(f" - {msg}") - line["todo"].append( - f"merge or close conflict PR {rm.pr_link(pr[0])}" - + ( - ", then Take RC again" - if newer == branch - else f" on `{newer}`" - ) - if pr - else msg.split(": ", 1)[1] - ) + message = functools.partial( + missing_message, + repo, + branch, + target, + older, + newer, + annotate=annotate, + failure=failure, + ) + for row in rows_out: + _row_out( + repo, + main, + refs, + ref, + lookup, + message, + annotate, + branch, + older, + newer, + row, + out, + line, + ) out.append("") if oss_repo: n, o_out, o_lines = oss_pins( @@ -412,65 +468,73 @@ def next_steps(lines, order): ) +def _cell(parts, none="-"): + """A table cell: the parts on separate lines, pipes escaped.""" + return "
".join(parts).replace("|", "\\|") or none + + +def _fixes(name, ls): + """A fixes row of the checks table for the lines ls.""" + # Accepted: an UNCLASSIFIABLE-only line has no todo, so its row reads ✅ (the detail still counts it). + named = [x for x in ls if x["state"] == "stabilizing" or x["todo"] or x["rbo"]] + rest = [x for x in ls if x not in named] + counts = [f"`{x['line']}` {x['summary']}" for x in named] + if rest: + side = rest[0]["side"] if len({x["side"] for x in rest}) == 1 else "other" + counts.append( + f"{len(rest)} {side} line{'s' * (len(rest) > 1)} converged ({sum(x['picks'] for x in rest)} picks)" + ) + todo = next_steps(ls, []) + return ( + "⛔" if todo else "✅", + name, + _cell(counts, "no other supported line"), + _cell(todo), + ) + + +def _order_row(order): + """The promote order row of the checks table.""" + return ( + "⛔" if order else "✅", + "promote order", + _cell([o.split(":")[0] for o in order], "no older line is waiting to go GA"), + _cell(next_steps([], order)), + ) + + +def _rbo_row(lines): + """The release-branch-only row of the checks table.""" + rbo = [r for x in lines for r in x["rbo"]] + return ( + "⚠" if rbo else "✅", + "release-branch-only to check", + _cell(rbo, "none"), + "check whether the newer line needs its own equivalent" if rbo else "-", + ) + + def checks(lines, order, out, split=False): """(checks table, folded gate detail) from gate()'s lines and out and relstate.promote_blockers' order. split: one fixes row per direction (Promote) instead of one for every other line (the Take RC preview). Blocking and to-check rows name what to do; healthy rows show counts only. A line is named only when it is stabilizing for the target or has something to do or check; the rest are one trailing count. """ - - def cell(parts, none="-"): - return "
".join(parts).replace("|", "\\|") or none - - def fixes(name, ls): - # Accepted: a line whose only finding is UNCLASSIFIABLE has no todo, so it is counted as converged here and - # the row reads ✅. The line's summary in the folded gate detail (out) still counts them. - named = [x for x in ls if x["state"] == "stabilizing" or x["todo"] or x["rbo"]] - rest = [x for x in ls if x not in named] - counts = [f"`{x['line']}` {x['summary']}" for x in named] - if rest: - side = rest[0]["side"] if len({x["side"] for x in rest}) == 1 else "other" - counts.append( - f"{len(rest)} {side} line{'s' * (len(rest) > 1)} converged ({sum(x['picks'] for x in rest)} picks)" - ) - todo = next_steps(ls, []) - return ( - "⛔" if todo else "✅", - name, - cell(counts, "no other supported line"), - cell(todo), - ) - own, pinned = ( [x for x in lines if not x.get("oss")], [x for x in lines if x.get("oss")], ) if split: rows = [ - fixes(f"fixes match {side} lines", [x for x in own if x["side"] == side]) + _fixes(f"fixes match {side} lines", [x for x in own if x["side"] == side]) for side in ("newer", "older") ] else: - rows = [fixes("fixes match the other lines", own)] + rows = [_fixes("fixes match the other lines", own)] if pinned: - rows.append(fixes("OSS pins match the other lines", pinned)) - rows.append( - ( - "⛔" if order else "✅", - "promote order", - cell([o.split(":")[0] for o in order], "no older line is waiting to go GA"), - cell(next_steps([], order)), - ) - ) - rbo = [r for x in lines for r in x["rbo"]] - rows.append( - ( - "⚠" if rbo else "✅", - "release-branch-only to check", - cell(rbo, "none"), - "check whether the newer line needs its own equivalent" if rbo else "-", - ) - ) + rows.append(_fixes("OSS pins match the other lines", pinned)) + rows.append(_order_row(order)) + rows.append(_rbo_row(lines)) table = ["| check | result | do this |", "|---|---|---|"] + [ f"| {i} {c} | {r} | {d} |" for i, c, r, d in rows ] @@ -479,10 +543,11 @@ def fixes(name, ls): def main(argv=None): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - p.add_argument("--branch", required=True) + p.add_argument("--branch", type=rm.option_free, required=True) p.add_argument("--target", required=True) p.add_argument( "--promote-ref", + type=rm.option_free, help="the RC being built or promoted; default: the branch's own reference", ) p.add_argument("--state", help="relstate JSON; default: derive it now") @@ -505,12 +570,13 @@ def main(argv=None): ) a = p.parse_args(argv) try: + out_file = a.out and rm.local_path(a.out) if rs.PINS and not a.oss_repo: raise RuntimeError( f"--oss-repo (or OSS_REPO) is required: this repo pins {rs.PINS}" ) if a.state: - with open(a.state) as f: + with open(rm.local_path(a.state)) as f: state = json.load(f) else: state = rs.derive(a.repo, dt.datetime.now(dt.UTC)) @@ -529,9 +595,9 @@ def main(argv=None): print(f"::error::convgate: {type(e).__name__}: {e}") return 1 print("\n".join(out + order)) - if a.out: + if out_file: table, detail = checks(lines, order, out) - with open(a.out, "a") as f: + with open(out_file, "a") as f: f.write("\n".join(table + [""] + detail) + "\n\n") return 2 if code or order else 0 diff --git a/.github/release/oss.py b/.github/release/oss.py index dd6f1b9a36..33bc2a1df5 100644 --- a/.github/release/oss.py +++ b/.github/release/oss.py @@ -30,6 +30,7 @@ PREFIX = rs.REPOS[OSS.split("/")[1].lower()][4] VER = rs.ver_re(PREFIX) PIN_RE = re.compile(r'^unstract-platform-version = "([^"]*)"$', re.M) +RELEASE = "release/" # release branch name prefix HERE = os.path.dirname(os.path.abspath(__file__)) @@ -51,27 +52,14 @@ def line_of(tag): def tip(oss, branch): - r = rm.git( - oss, - "rev-parse", - "-q", - "--verify", - f"refs/remotes/origin/{branch}^{{commit}}", - check=False, - ) - return r.stdout.strip() if r.returncode == 0 else None + return rm.resolve(oss, f"refs/remotes/origin/{branch}") def tags_on(oss, sha, branch): """Version tags of branch's minor on commit sha.""" - line = branch.removeprefix("release/") - return [ - t - for t in rm.git( - oss, "tag", "--points-at", sha, "-l", f"{PREFIX}v{line}.*" - ).split() - if (m := VER.fullmatch(t)) and f"{m[1]}.{m[2]}" == line - ] + return rs.line_tags( + oss, branch.removeprefix(RELEASE), "--points-at", sha, prefix=PREFIX + ) def choose(tags): @@ -122,10 +110,8 @@ def _tag_push(oss, name, sha, msgs, *refspecs): def _bump(oss, branch, base, tag, slack_url, old): - """One empty bot commit `release: ` on base, tagged, pushed as branch's new tip with its tag. The commit gives - the RC its own commit, so a line never shares a tag's commit with main or another line. The push is leased on - old, the branch tip it read: for rc, a branch deleted or moved since then fails it instead of being re-created or - overwritten; for cut, old is "" (no branch), so a branch created since then fails it. + """Push one empty bot commit `release: ` on base as branch's new tip, with its tag. Leased on old (the tip + read; "" for a cut), so a concurrent change fails the push. """ sha = _bot( oss, @@ -150,18 +136,17 @@ def _bump(oss, branch, base, tag, slack_url, old): def gate(oss, branch, ref, annotate): - """The OSS-line convergence gate of branch at tag ref, run on the OSS clone as the OSS repo: 0 pass, 2 block, - 1 error. Its checks table goes to the step summary. argparse also exits 2, on a usage error: that is an error, - never a block a force could override. - """ + """Run convgate on the OSS clone: 0 pass, 2 block, 1 error (an argparse usage exit 2 counts as an error).""" if summary := os.environ.get("GITHUB_STEP_SUMMARY"): with open(summary, "a") as f: f.write(f"#### OSS line `{branch}` at `{ref}`\n\n") + oss, branch, ref = rm.local_path(oss), rm.option_free(branch), rm.option_free(ref) env = { **os.environ, "GITHUB_REPOSITORY": OSS, "GH_TOKEN": os.environ.get("OSS_TOKEN") or os.environ.get("GH_TOKEN", ""), } + # convgate stdout -> our stderr: our stdout is the tag the workflow reads. r = subprocess.run( [ sys.executable, @@ -181,7 +166,7 @@ def gate(oss, branch, ref, annotate): stdout=2, stderr=subprocess.PIPE, text=True, - ) # stdout is the tag the workflow captures + ) sys.stderr.write(r.stderr) if ( r.returncode not in (0, 2) @@ -252,7 +237,7 @@ def reopen(oss, branch): stabilizing again and its next main push catches up the plain fixes it skipped. Promote writes the tag again at the line's last GA. A missing tag is a no-op. Summary lines. """ - mark = rs.STABILIZED + branch.removeprefix("release/") + mark = rs.STABILIZED + branch.removeprefix(RELEASE) if not rs._tag_commit(oss, mark): return [] _push(oss, f":refs/tags/{mark}") @@ -286,6 +271,25 @@ def promote(repo, oss, cloud_rc, force=False, reason="", stabilized=False): return stable +def _conflicts(branch, force): + """(open backport conflict PR refs on OSS branch, unchecked notes); a failed listing raises unless force.""" + unchecked = [] + try: + prs = [ + f"#{p['number']}" + for p in bp.conflict_prs(OSS, branch, os.environ.get("OSS_TOKEN")) + ] + except RuntimeError as e: + if not force: + raise + print( + f"::warning::Could not list open backport conflict PRs on OSS {branch} (forced on): {e}", + file=sys.stderr, + ) + prs, unchecked = [], ["conflict PRs unchecked: listing failed"] + return prs, unchecked + + def _promote(repo, oss, cloud_rc, force, reason): """The stable OSS tag of the OSS RC cloud_rc pins, tagging it when it has none yet. Open backport conflict PRs on the OSS line (or a failed listing of them) and its gate block first unless force; the other cloud product finds @@ -304,22 +308,7 @@ def _promote(repo, oss, cloud_rc, force, reason): raise RuntimeError(f"OSS {e}") from None if rs._tag_commit(oss, stable) == sha: return stable # promoted earlier, e.g. by the other cloud product - try: - prs, unchecked = ( - [ - f"#{p['number']}" - for p in bp.conflict_prs(OSS, branch, os.environ.get("OSS_TOKEN")) - ], - [], - ) - except RuntimeError as e: - if not force: - raise - print( - f"::warning::Could not list open backport conflict PRs on OSS {branch} (forced on): {e}", - file=sys.stderr, - ) - prs, unchecked = [], ["conflict PRs unchecked: listing failed"] + prs, unchecked = _conflicts(branch, force) if prs and not force: raise RuntimeError( f"OSS {branch} has open backport conflict PRs {', '.join(prs)}: resolve and merge or close " @@ -355,7 +344,7 @@ def retire(oss, branch, reason, dry_run=True): (no OSS GA) its RC tags and the branch. With a GA its tags stay, stabilized/X.Y with them, and the minor is never reused; without one it is. Summary lines. """ - line = branch.removeprefix("release/") + line = branch.removeprefix(RELEASE) sha = tip(oss, branch) if not sha: done = rm.git(oss, "tag", "-l", f"retired/{line}-*").split() @@ -390,9 +379,7 @@ def retire(oss, branch, reason, dry_run=True): ] if dry_run: return out - # One atomic push leased on the tip read above: a commit pushed since then fails the whole retire, so on origin - # the branch, its RC tags and the archive tag all stay as they were. The archive tag is always sent (a no-op when - # origin has it), so a re-run in this clone archives the tip too. + # Atomic and leased on the tip read above; the archive tag is always sent so a re-run in this clone archives too. _tag_push( oss, archive, @@ -415,13 +402,13 @@ def main(argv=None): ) sub = p.add_subparsers(dest="cmd", required=True) c = sub.add_parser("cut") - c.add_argument("--line", required=True) + c.add_argument("--line", type=rm.option_free, required=True) c.add_argument("--slack-url", default="") r = sub.add_parser("rc") - r.add_argument("--branch", required=True) + r.add_argument("--branch", type=rm.option_free, required=True) r.add_argument("--slack-url", default="") m = sub.add_parser("promote") - m.add_argument("--rc", required=True) + m.add_argument("--rc", type=rm.option_free, required=True) m.add_argument("--force", action="store_true") m.add_argument("--reason", default="") m.add_argument("--stabilized", action="store_true") @@ -429,13 +416,12 @@ def main(argv=None): try: if not a.oss_repo: raise RuntimeError("--oss-repo (or OSS_REPO) is required") - tag = ( - cut(a.oss_repo, a.line, a.slack_url) - if a.cmd == "cut" - else rc(a.oss_repo, a.branch, a.slack_url) - if a.cmd == "rc" - else promote(".", a.oss_repo, a.rc, a.force, a.reason, a.stabilized) - ) + if a.cmd == "cut": + tag = cut(a.oss_repo, a.line, a.slack_url) + elif a.cmd == "rc": + tag = rc(a.oss_repo, a.branch, a.slack_url) + else: + tag = promote(".", a.oss_repo, a.rc, a.force, a.reason, a.stabilized) except RuntimeError as e: print(f"::error::{e}", file=sys.stderr) # stdout is the tag the workflow captures return 1 diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py index e1e2774edc..2ed997e550 100644 --- a/.github/release/relmgr.py +++ b/.github/release/relmgr.py @@ -1,36 +1,35 @@ """Pure-git checks for the release-branch strategy. -Every function reads a local repo only; nothing is stored, everything is derived from git. +Every function reads a local repo; nothing is stored. +Repo, state and output paths must resolve under the working directory, GITHUB_WORKSPACE, RUNNER_TEMP or the temp dir. """ import ast +import datetime as dt import os import re import subprocess +import tempfile BOT_EMAIL = "releases@unstract.com" +_FORMAT_BODY = "--format=%B" # git log: a commit's raw message PICK_RE = re.compile(r"\(cherry picked from commit ([0-9a-f]{7,40})\)") -# Anywhere in the message, at a line start; `* ` too, because a squash with COMMIT_MESSAGES lists each commit's -# subject as `* ` and the bot's Release-branch-only commit is subject-only. +# At any line start; also after "* ", as a COMMIT_MESSAGES squash lists each subject that way. RBO_RE = re.compile(r"^(?:\* )?Release-branch-only: (\S+)", re.M) -# `UN-1 [FIX] x` or `[FIX] x`: the ticket is optional. Known limit: one ticket only, so `UN-1 UN-2 [FIX] x` without -# a fix/ label fails Lint PR and is unclassifiable for Backport. +# `TICKET [FIX] x` or `[FIX] x`; the ticket is optional. Known limit: one ticket only; `A-1 B-2 [FIX] x` needs a +# fix/ label. PR_TITLE_RE = re.compile( r"^(?:(?P[A-Z][A-Z0-9]*-\d+) )?\[(?PFIX|HOT-FIX|FEAT|GATED-FEAT|DOCS|DEPS|MISC|DEV|WIP)\] \S" ) FIX_CLASSES = ("fix", "security", "critical") -REVERT_RE = re.compile( - r'^Revert "(.+)"' -) # GitHub's revert button; greedy, so a revert of a revert nests -REVERTS_RE = re.compile( - r"^This reverts commit ([0-9a-f]{7,40})", re.M -) # git revert's body line -DECLINED_RE = re.compile( - r"^Declined: (.+)$", re.M -) # the reason line Backport's record job writes on a decline commit -BACKPORT_HEAD_RE = re.compile( - r"backport/(\d+\.\d+)/([0-9a-f]{7})" -) # the head Backport opens a pick PR from: line, sha7; fullmatch +# GitHub's revert title; greedy, so a revert of a revert nests. +REVERT_RE = re.compile(r'^Revert "(.+)"') +# git revert's body line. +REVERTS_RE = re.compile(r"^This reverts commit ([0-9a-f]{7,40})", re.M) +# Reason line of a decline commit. +DECLINED_RE = re.compile(r"^Declined: (.+)$", re.M) +# Pick-PR head: backport// (use fullmatch). +BACKPORT_HEAD_RE = re.compile(r"backport/(\d+\.\d+)/([0-9a-f]{7})") def _web(): @@ -55,10 +54,6 @@ def tag_link(tag): return f"[`{tag}`]({_web()}/releases/tag/{tag})" if _web() else f"`{tag}`" -def run_link(run_id): - return f"[run]({_web()}/actions/runs/{run_id})" if _web() else f"run {run_id}" - - def image_link(target, tag): """The Artifact Registry console page of target's repository (RELSTATE_AR_), the tag in the text; plain text when that variable is unset. @@ -73,9 +68,40 @@ def image_link(target, tag): ) -_FULL = ( - set() -) # repos confirmed non-shallow; a failed check is retried, never cached as a pass +_FULL = set() # repos confirmed non-shallow; failures are not cached + + +class Refused(RuntimeError, ValueError): + """An argument refused by a guard: a RuntimeError for the CLI handlers, a ValueError for argparse `type=`.""" + + +def option_free(value): + """value, refused when git or a subprocess would read it as an option.""" + if value.startswith("-"): + raise Refused(f"{value!r} starts with '-'") + return value + + +def local_path(path): + """path, refused unless it is the runner's step summary file or resolves under the working directory, + GITHUB_WORKSPACE, RUNNER_TEMP or the temp dir. + """ + if path == os.environ.get("GITHUB_STEP_SUMMARY"): + return path + real = os.path.realpath(option_free(path)) + roots = { + os.path.realpath(r) + for r in ( + os.getcwd(), + os.environ.get("GITHUB_WORKSPACE"), + os.environ.get("RUNNER_TEMP"), + tempfile.gettempdir(), + ) + if r + } + if not any(os.path.commonpath([real, r]) == r for r in roots): + raise Refused(f"{path} is outside the workspace and temp dirs") + return path def _require_full_history(repo): @@ -97,6 +123,7 @@ def _require_full_history(repo): def git(repo, *args, check=True): + repo = local_path(repo) _require_full_history(repo) r = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True) if check and r.returncode: @@ -106,6 +133,34 @@ def git(repo, *args, check=True): return r.stdout.strip() if check else r +def resolve(repo, rev): + """Full sha of the commit rev names, or None.""" + r = git(repo, "rev-parse", "-q", "--verify", f"{rev}^{{commit}}", check=False) + return r.stdout.strip() if r.returncode == 0 else None + + +def message(repo, sha): + """The raw commit message of sha.""" + return git(repo, "log", "-1", _FORMAT_BODY, sha) + + +def subject_pr(repo, sha, check=True): + """The `(#n)` at the end of sha's subject, else None. check=False: an unknown sha reads as None.""" + r = git(repo, "log", "-1", "--format=%s", sha, check=check) + m = re.search(r"\(#(\d+)\)$", r if check else r.stdout.strip()) + return int(m[1]) if m else None + + +def same_sha(a, b): + """Do a and b name the same commit, either one abbreviated?""" + return a.startswith(b) or b.startswith(a) + + +def ts(s): + """An ISO-8601 time; a trailing Z reads as UTC.""" + return dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + + def is_ancestor(repo, a, b): r = git(repo, "merge-base", "--is-ancestor", a, b, check=False) if r.returncode not in (0, 1): # 1 = "no"; anything else is an error, never a "no" @@ -115,8 +170,8 @@ def is_ancestor(repo, a, b): def on_main(repo, sha, main="main"): """Does `sha` (full or unambiguous prefix) name a commit on `main`?""" - r = git(repo, "rev-parse", "-q", "--verify", f"{sha}^{{commit}}", check=False) - return r.returncode == 0 and is_ancestor(repo, r.stdout.strip(), main) + full = resolve(repo, sha) + return full is not None and is_ancestor(repo, full, main) def _is_empty(repo, sha): @@ -130,11 +185,7 @@ def picked_from(repo, sha): them see a hand-written short sha and the bot's full one as the same original. A trailer that does not resolve (unknown or ambiguous) is kept as written. """ - out = [] - for o in PICK_RE.findall(git(repo, "log", "-1", "--format=%B", sha)): - r = git(repo, "rev-parse", "-q", "--verify", f"{o}^{{commit}}", check=False) - out.append(r.stdout.strip() if r.returncode == 0 else o) - return out + return [resolve(repo, o) or o for o in PICK_RE.findall(message(repo, sha))] def branch_range(repo, branch, main): @@ -147,16 +198,16 @@ def branch_range(repo, branch, main): def range_messages(repo, branch, main): """All commit messages in merge-base(main, branch)..branch, as one string (one git call).""" return git( - repo, "log", "--format=%B", f"{git(repo, 'merge-base', main, branch)}..{branch}" + repo, "log", _FORMAT_BODY, f"{git(repo, 'merge-base', main, branch)}..{branch}" ) -# 1. Commit kinds / release-branch CI gate --------------------------------------------------------------- +# Commit kinds / release-branch CI gate --------------------------------------------------------------- def reverted_pick(repo, sha, main="main"): """The full sha of the pick that `sha` reverts on its own branch (after the cut), else None.""" - m = REVERTS_RE.search(git(repo, "log", "-1", "--format=%B", sha)) + m = REVERTS_RE.search(message(repo, sha)) if not m or git(repo, "cat-file", "-e", f"{m[1]}^{{commit}}", check=False).returncode: return None t = git(repo, "rev-parse", f"{m[1]}^{{commit}}") @@ -167,20 +218,16 @@ def reverted_pick(repo, sha, main="main"): def classify_commit(repo, sha, bot_email=BOT_EMAIL, main="main"): - body = git(repo, "log", "-1", "--format=%B", sha) + body = message(repo, sha) if PICK_RE.search(body): if not all(on_main(repo, o, main) for o in PICK_RE.findall(body)): - return ( - "invalid" # trailer names nothing on main: not a pick of a main original - ) + return "invalid" # trailer names nothing on main return "declined" if _is_empty(repo, sha) else "cherry-pick" - if reverted_pick( - repo, sha, main - ): # backing a pick out of one branch declines it there + # A revert of a pick declines it on this branch. + if reverted_pick(repo, sha, main): return "declined" - if RBO_RE.search( - body - ): # before the author check: a bot-made pin bump is still owed a report + # Checked before the author: a bot-made pin bump still needs a report. + if RBO_RE.search(body): return "release-branch-only" # Trust assumption: the author email is client-set; the review every release-branch PR gets is the backstop. if git(repo, "log", "-1", "--format=%ae", sha).lower() == bot_email.lower(): @@ -189,14 +236,14 @@ def classify_commit(repo, sha, bot_email=BOT_EMAIL, main="main"): def pr_commits(repo, base, head_ref, head="HEAD"): - """The commits the PR check classifies on a PR from head_ref into base. A backport/* PR lands as its squash, whose - message keeps the first commit's pick trailer: only that commit is judged, so a plain resolution commit after it - passes. This relies on release/* allowing squash merges only (a ruleset, set before Backport runs); a rebase - merge would land the unjudged commits as they are. Any other PR: every commit in base..head. + """Commits the PR check classifies. A backport/* PR lands as one squash, so only its first commit is judged + (needs squash-only merges on release/*). Any other PR: every commit in base..head. """ out = git(repo, "rev-list", "--reverse", f"{base}..{head}") commits = out.split() if out else [] - return commits[:1] if is_backport(head_ref) else commits + if not commits or not is_backport(head_ref): + return commits + return [commits[0]] def is_backport(head_ref): @@ -219,70 +266,60 @@ def conflict_markers(repo, rev="HEAD"): return [m.split(":", 1)[1] for m in r.stdout.splitlines()] -def ci_commits( - repo, branch, event, before="", head_ref="", labelled=False, main="origin/main" -): - """Release branch CI over one event: ([(sha, kind)], where). ValueError: conflict markers in a backport PR's tree. - - pull_request: the PR's own commits (a backport PR's first only, invalid unless it carries exactly one pick - trailer, naming the head's sha7); labelled turns its invalid commits into - release-branch-only, since the squash at merge writes the line. push: the pushed commits before..HEAD, or the - whole branch when before is all zeros (branch creation) or gone (a force push). - """ +def _ci_scope(repo, branch, event, before, head_ref, main): + """ci_commits' ([sha], where) for one event.""" if event == "pull_request": if is_backport(head_ref) and (marked := conflict_markers(repo)): raise ValueError( f"conflict markers in {', '.join(marked)}: resolve before merging" ) - commits, where = pr_commits(repo, branch, head_ref), f"PR into {branch}" - elif ( + return pr_commits(repo, branch, head_ref), f"PR into {branch}" + if ( set(before) - {"0"} and git(repo, "cat-file", "-e", f"{before}^{{commit}}", check=False).returncode == 0 ): - # Only the pushed commits. Classifying the whole branch would keep every later push red - # after one invalid commit landed, so a revert could never clear it. The PR check covers PR commits. + # Pushed commits only, so a revert can clear an earlier invalid commit. out = git(repo, "rev-list", "--reverse", f"{before}..HEAD") - commits, where = out.split() if out else [], f"push to {branch}" - else: - commits, where = branch_range(repo, branch, main), branch + return (out.split() if out else []), f"push to {branch}" + return branch_range(repo, branch, main), branch + + +def ci_commits( + repo, branch, event, before="", head_ref="", labelled=False, main="origin/main" +): + """Release branch CI over one event: ([(sha, kind)], where). ValueError: conflict markers in a backport PR's tree. + + pull_request: the PR's own commits (a backport PR's first only, invalid unless it carries exactly one pick + trailer, naming the head's sha7); labelled turns its invalid commits into + release-branch-only, since the squash at merge writes the line. push: the pushed commits before..HEAD, or the + whole branch when before is all zeros (branch creation) or gone (a force push). + """ + commits, where = _ci_scope(repo, branch, event, before, head_ref, main) labelled = labelled and event == "pull_request" + backport_pr = event == "pull_request" and is_backport(head_ref) rows = [] for c in commits: kind = classify_commit(repo, c, main=main) - # The squash keeps every trailer of a backport PR's first commit: one naming the head's fix, or it records others. - if ( - event == "pull_request" - and is_backport(head_ref) - and not pick_trailer(head_ref, git(repo, "log", "-1", "--format=%B", c)) - ): + # The squash keeps the first commit's trailers: it must name the head's fix only. + if backport_pr and not pick_trailer(head_ref, message(repo, c)): kind = "invalid" - # Known limit: the label passes every invalid commit; it does not check that a Release-branch-only line will - # be in the squash (a fork PR, or a merge before the bot pushes that commit). + # Known limit: the label passes every invalid commit without checking that the squash will carry a + # Release-branch-only line. rows.append( (c, "release-branch-only (label)" if labelled and kind == "invalid" else kind) ) return rows, where -def check_branch(repo, branch, main="main"): - """Release-branch CI gate: the invalid commits on the branch. Empty list == pass.""" - return [ - c - for c in branch_range(repo, branch, main) - if classify_commit(repo, c, main=main) == "invalid" - ] - - -# 2. PR class (Lint PR) ------------------------------------------------------ +# PR class (Lint PR) ------------------------------------------------------ FIX_LABELS = ("fix/security", "fix/critical", "fix/needs-backport") def pr_fix_class(title, labels=()): """'fix' | 'security' | 'critical' for fix PRs, None for non-fix PRs; ValueError = CI rejects.""" - # Anything that looks like a class label (fix/, the old fix-class/, near-typos) but is - # not one of FIX_LABELS: a typo must not silently drop or downgrade the class. + # Reject near-miss class labels (fix/, fix-class/, typos) so a typo cannot drop the class. bad = [ l for l in labels @@ -296,8 +333,7 @@ def pr_fix_class(title, labels=()): } if len(classes) > 1: raise ValueError(f"conflicting fix labels: {sorted('fix/' + c for c in classes)}") - # A revert takes the reverted PR's class: it must follow the original onto every branch that carries it. - # Stripped: GitHub keeps a title's leading space, and a typo'd space must not leave a merged fix unclassifiable. + # A revert takes the reverted PR's class. Strip: GitHub keeps a leading space. inner = title.strip() while r := REVERT_RE.match(inner): inner = r.group(1).strip() @@ -336,7 +372,7 @@ def rbo_key(title, pr): return m["ticket"] if m and m["ticket"] else f"#{pr}" -# 3. Version allocation (at build time; a version names one commit) ------------------------------------------------ +# Version allocation (at build time; a version names one commit) ------------------------------------------------ def _line(branch): @@ -373,10 +409,8 @@ def rc_version(repo, branch, prefix="", taken=()): """Tag for a build of the branch tip, allocated from tags at build time. If the tip already carries an RC tag of this line, that tag is returned: a commit gets one RC tag. - taken: tag names already used elsewhere (every target's registry tags). An RC there is never allocated again: - an artifact pushed by a run whose tag push failed would otherwise block the line. The one exception is a tip - that is already the bot's bump for it (already_bumped): that run is resumed. - Needs the full tag set locally (fetch tags in CI): with no tags every line reads as freshly cut. + taken: tags already pushed to a registry; never reallocated, except for a tip that is already the bot's bump + for that tag (resumed). Needs all tags fetched. """ x, y = _line(branch) for t in git(repo, "tag", "--points-at", branch).split(): @@ -401,10 +435,8 @@ def rc_version(repo, branch, prefix="", taken=()): def already_bumped(repo, branch, tag): - """True when the branch tip is the bot's bump for tag: build-generated with subject `release: chart ` - and the chart at the version, or, in a repo with no chart, subject `release: ` (an empty commit). - That tip is left behind when the bump was pushed but the run stopped before the tag push (a failed build); - Take RC build then tags it instead of committing another bump. + """Is the branch tip the bot's bump for tag (`release: chart ` with the chart at , or `release: ` with + no chart)? Left behind when a run pushed the bump but not the tag. """ ver = tag.removeprefix("v") chart = git(repo, "show", f"{branch}:{CHART}", check=False) @@ -418,18 +450,12 @@ def already_bumped(repo, branch, tag): def promote_version(repo, rc_tag): - """Stable tag for promoting rc_tag: the commit's existing stable tag, else one past the highest. - - Stable Z follows commit order, so an upgrade never loses a fix. The Z can differ from the - RC's (a target that passes a later RC takes the next patch, so a version names one commit; the - RC->stable mapping goes in the release body), which departs from the spec's - x.y.(z+1)-rc.N -> vx.y.(z+1) when a promote lands while a later patch is in RC. An RC older than a - commit that already has a stable tag is refused as a downgrade. Both are harness choices, open - with the spec owner. + """Stable tag for rc_tag: the commit's existing stable tag, else one past the highest. Z follows commit order, + so it can differ from the RC's Z (the release body maps RC to stable). An RC older than an already-stable commit + is refused as a downgrade. """ - m = re.fullmatch( - r"((?:[a-z]+-)?)v(\d+)\.(\d+)\.(\d+)-rc\.\d+", rc_tag - ) # the stable tag keeps the RC's prefix + # The stable tag keeps the RC's prefix. + m = re.fullmatch(r"((?:[a-z]+-)?)v(\d+)\.(\d+)\.(\d+)-rc\.\d+", rc_tag) if not m: raise ValueError(f"not an RC tag: {rc_tag}") p, x, y = m.group(1), m.group(2), m.group(3) @@ -446,19 +472,16 @@ def promote_version(repo, rc_tag): return f"{p}v{x}.{y}.{max(stable) + 1 if stable else 0}" -# 4. Convergence -------------------------------------------------------------------- +# Convergence -------------------------------------------------------------------- def is_converged(repo, c_main, branch, main="main"): - """Inherited (ancestor) or picked/declined after the cut (trailer in merge-base..branch). - Known limit: no test covers the inherited path (a commit from before the cut) on its own. - """ + """Inherited (ancestor) or picked/declined after the cut (trailer in merge-base..branch).""" full = git(repo, "rev-parse", f"{c_main}^{{commit}}") if is_ancestor(repo, full, branch): return True return any( - o.startswith(full) or full.startswith(o) - for o in PICK_RE.findall(range_messages(repo, branch, main)) + same_sha(o, full) for o in PICK_RE.findall(range_messages(repo, branch, main)) ) @@ -491,15 +514,20 @@ def decline_records(repo, branch, main="main"): for c in branch_range(repo, branch, main): kind = classify_commit(repo, c, main=main) if kind in ("cherry-pick", "declined"): - rp = reverted_pick(repo, c, main) - body = git(repo, "log", "-1", "--format=%B", c) - reasons = DECLINED_RE.findall(body) - reason = reasons[-1].strip() if reasons and not rp else body.splitlines()[0] - for o in picked_from(repo, rp or c): - rec[o] = (c, reason) if kind == "declined" else None + _record_decline(repo, c, kind, main, rec) return {o: r for o, r in rec.items() if r} +def _record_decline(repo, c, kind, main, rec): + """decline_records' step for one pick-trailer commit c: set rec[original] for each original it names.""" + rp = reverted_pick(repo, c, main) + body = message(repo, c) + reasons = DECLINED_RE.findall(body) + reason = reasons[-1].strip() if reasons and not rp else body.splitlines()[0] + for o in picked_from(repo, rp or c): + rec[o] = (c, reason) if kind == "declined" else None + + def shipped_prs(repo, tags, main="main", lookup=None, line=None): """PR numbers whose main original was cherry-picked (declines excluded) into any of `tags`: the changelog drops them. @@ -519,23 +547,29 @@ def shipped_prs(repo, tags, main="main", lookup=None, line=None): } prs = set() for o in originals: - m = re.search(r"\(#(\d+)\)$", git(repo, "log", "-1", "--format=%s", o)) - n = int(m[1]) if m else lookup(o) if lookup else None + n = subject_pr(repo, o) + if n is None and lookup: + n = lookup(o) if n is None: raise RuntimeError( f"no PR number for {o[:12]} (picked into {', '.join(tags)})" ) prs.add(n) - for c in branch_range(repo, line, main) if line else []: + if line: + prs |= _line_pick_prs(repo, line, originals, main, lookup) + return prs + + +def _line_pick_prs(repo, line, originals, main, lookup): + """shipped_prs' PRs that landed line's picks of `originals`.""" + prs = set() + for c in branch_range(repo, line, main): if classify_commit(repo, c, main=main) == "cherry-pick" and any( - o.startswith(d) or d.startswith(o) - for o in originals - for d in picked_from(repo, c) - ): # may be abbreviated - m = re.search(r"\(#(\d+)\)$", git(repo, "log", "-1", "--format=%s", c)) + same_sha(o, d) for o in originals for d in picked_from(repo, c) + ): prs |= { n - for n in (int(m[1]) if m else None, lookup(c) if lookup else None) + for n in (subject_pr(repo, c), lookup(c) if lookup else None) if n is not None } return prs @@ -544,23 +578,12 @@ def shipped_prs(repo, tags, main="main", lookup=None, line=None): def convergence_report(repo, older, newer, main="main", newer_branch=None): """[(sha_on_older, status)] with status CONVERGED | MISSING | UNCLASSIFIABLE | RBO-ABSENT. Blocks iff any MISSING. - RBO-ABSENT is report-only: a Release-branch-only key (the ticket, or `#`) on `older` with no - Release-branch-only commit for that key on `newer`. Case 5 (config/migration) owes nothing; - a security fix for code gone from main may need its own equivalent on `newer` if the vulnerable code is still - there, so a human looks. - - Both scan directions are this one call: promoting `newer` scans each older branch into it, - promoting `older` scans it into each newer branch; either way it is older's picks vs newer. - Pass `newer` as a branch while it is in RC. Once it is GA, pass its latest stable tag (for - the promoting target) as `newer`, since "shipped in its latest patch" is not the branch tip, and - the branch as `newer_branch`: a branch need only carry a decline, not ship it. - - A pick whose original cannot be placed on `main` fails release-branch CI and is MISSING here: it is never - report-only, and never CONVERGED (a branch-first fix has no main original to converge on). + RBO-ABSENT is report-only (a key on older with no Release-branch-only commit on newer). Pass newer as a branch + while in RC, or as its latest stable tag once GA, with the branch as newer_branch (it need only carry a + decline). A pick whose original is not on main is MISSING. """ out = [] - # Newest trailer wins on both sides: a decline on newer that a later pick undid is not a decline, and a pick on - # older that a later decline or revert backed out is not owed. + # Newest trailer wins on both sides (see trailer_states). declined = declined_originals(repo, newer_branch or newer, main) newer_picks = { o @@ -569,47 +592,90 @@ def convergence_report(repo, older, newer, main="main", newer_branch=None): } older_declined = declined_originals(repo, older, main) newer_rbo = set(RBO_RE.findall(range_messages(repo, newer, main))) - same = lambda o, names: any( - o.startswith(d) or d.startswith(o) for d in names - ) # trailers may be abbreviated for c in branch_range(repo, older, main): kind = classify_commit(repo, c, main=main) - if kind == "invalid": - out.append((c, "MISSING" if picked_from(repo, c) else "UNCLASSIFIABLE")) - elif kind == "cherry-pick": - if all( - same(o, older_declined) for o in picked_from(repo, c) - ): # backed out of `older`: newer owes nothing - continue - ok = all( - is_ancestor(repo, git(repo, "rev-parse", f"{o}^{{commit}}"), newer) - or same(o, newer_picks) - or same(o, declined) - for o in picked_from(repo, c) - ) - out.append((c, "CONVERGED" if ok else "MISSING")) - elif kind == "release-branch-only": - body = git(repo, "log", "-1", "--format=%B", c) - if not set(RBO_RE.findall(body)) & newer_rbo: - out.append((c, "RBO-ABSENT")) - # build-generated: excluded from convergence. - # declined on `older`: older does not carry the fix, so newer owes nothing. + status = _older_status( + repo, c, kind, newer, older_declined, newer_picks, declined, newer_rbo + ) + if status: + out.append((c, status)) return out -# 5. Relevance (advisory: does the branch still have the code a fix touches?) ------------------ +def _older_status(repo, c, kind, newer, older_declined, newer_picks, declined, newer_rbo): + """convergence_report's status for commit c on older, None when it is not reported.""" + if kind == "invalid": + return "MISSING" if picked_from(repo, c) else "UNCLASSIFIABLE" + if kind == "cherry-pick": + return _pick_status(repo, c, newer, older_declined, newer_picks, declined) + if kind == "release-branch-only": + body = message(repo, c) + if not set(RBO_RE.findall(body)) & newer_rbo: + return "RBO-ABSENT" + # build-generated: excluded from convergence. + # declined on `older`: older does not carry the fix, so newer owes nothing. + return None + + +def _same(o, names): + return any(same_sha(o, d) for d in names) + + +def _pick_status(repo, c, newer, older_declined, newer_picks, declined): + """convergence_report's status for a pick on older: CONVERGED | MISSING, None when it owes nothing.""" + # Backed out of older: newer owes nothing. + if all(_same(o, older_declined) for o in picked_from(repo, c)): + return None + ok = all( + is_ancestor(repo, git(repo, "rev-parse", f"{o}^{{commit}}"), newer) + or _same(o, newer_picks) + or _same(o, declined) + for o in picked_from(repo, c) + ) + return "CONVERGED" if ok else "MISSING" + + +# Relevance (advisory: does the branch still have the code a fix touches?) ------------------ def relevance(repo, fix_sha, branch): - """Are the exact line versions a main fix edits on `branch`? relevant | not-relevant | partial | unknown. - - Blames the pre-image lines the fix modifies/deletes in fix^; each last-modifying commit counts - if it is an ancestor of `branch` or was picked onto it. 'not-relevant' means those line versions - are absent, not that the code is: a line rewritten on main after the cut still exists on the - branch in its older form (adapt the pick by hand, never refactor). Advisory only. - Pure additions have no pre-image: 'unknown' (fallback, not implemented: treat as relevant iff - the touched files exist on the branch). + """Are the line versions a main fix edits on branch? relevant | not-relevant | partial | unknown. Blames the + fix's pre-image lines; each introducer counts if it is an ancestor of branch or picked onto it. Pure additions: + unknown. Advisory only. """ + introducers = _introducers(repo, fix_sha) + if not introducers: + return "unknown" + # Known limit: scans the branch log once per call. + picked = { + o + for c in git( + repo, "log", "--format=%H", "-F", "--grep=(cherry picked from commit", branch + ).split() + if not _is_empty(repo, c) + for o in picked_from(repo, c) + } # a decline carries nothing + hits = [ + is_ancestor(repo, c, branch) or any(c.startswith(p) for p in picked) + for c in introducers + ] + if all(hits): + return "relevant" + if not any(hits): + return "not-relevant" + return "partial" + + +def _minus_path(line): + """The path of a `--- ` file-header line, None for /dev/null.""" + p = line[4:].rstrip("\t") # git tab-suffixes paths with spaces + if p.startswith('"'): # C-quoted (", \, control chars): unquote to the raw bytes + p = ast.literal_eval("b" + p).decode() + return None if p == "/dev/null" else p[2:] + + +def _introducers(repo, fix_sha): + """The last-modifying commits of the pre-image lines fix_sha modifies/deletes (for relevance).""" parent, introducers, path, header = f"{fix_sha}^", set(), None, False diff = git( repo, @@ -626,15 +692,9 @@ def relevance(repo, fix_sha, branch): for line in diff.splitlines(): if line.startswith("diff --git "): path, header = None, True - elif header and line.startswith( - "--- " - ): # only in the file header: `--- x` in a hunk is content - p = line[4:].rstrip("\t") # git tab-suffixes paths with spaces - if p.startswith( - '"' - ): # C-quoted (", \, control chars): unquote to the raw bytes - p = ast.literal_eval("b" + p).decode() - path = None if p == "/dev/null" else p[2:] + # Only in the file header: `--- x` in a hunk is content. + elif header and line.startswith("--- "): + path = _minus_path(line) elif line.startswith("@@"): header = False m = re.match(r"@@ -(\d+)(?:,(\d+))? ", line) @@ -644,19 +704,4 @@ def relevance(repo, fix_sha, branch): repo, "blame", "--porcelain", "-L", f"{start},+{n}", parent, "--", path ) introducers |= set(re.findall(r"^([0-9a-f]{40}) \d+ \d+", blame, re.M)) - if not introducers: - return "unknown" - # ponytail: scans the whole branch log once per call; fine for fixtures, index trailers once if slow. - picked = { - o - for c in git( - repo, "log", "--format=%H", "-F", "--grep=(cherry picked from commit", branch - ).split() - if not _is_empty(repo, c) - for o in picked_from(repo, c) - } # a decline carries nothing - hits = [ - is_ancestor(repo, c, branch) or any(c.startswith(p) for p in picked) - for c in introducers - ] - return "relevant" if all(hits) else "not-relevant" if not any(hits) else "partial" + return introducers diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 2d20623a17..24bf369802 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -1,4 +1,4 @@ -"""Branch and target state for the V2 release workflows, derived from git + the target registries. +"""Branch and target state for the release-branch workflows, derived from git and the target registries. Usage: relstate.py [--repo DIR] [--now 2026-09-29T00:00:00Z] [--rc vX.Y.Z-rc.N] @@ -22,22 +22,18 @@ import relmgr as rm -# On-prem support window from the line's first stable chart push (its GA); a patch does not extend it. -WINDOW = dt.timedelta(days=184) # 184 days covers the longest 6-month span -WINDOW_LINES = ( - 6 # and only the newest 6 on-prem GA lines are supported, whichever ends first -) -# Per repo, so both repos run this same file: (targets, targets where only the latest GA line is supported, -# targets supported while the branch exists: retired only when another repo's Retire names the line, repo whose -# tags this repo's lines pin in pyproject.toml or None, tag prefix of this repo's lines). -# Artifacts are / in Artifact Registry; RELSTATE_AR_ names the repository (ar_repository). -# TARGETS: target -> (artifact holding RC / promoted builds, GA anchor artifact). Cloud GA = the vX.Y.Z retag -# of the build artifact; on-prem GA = the chart push. GIT instead of an artifact: the line's own git tags, the stable -# tag's date as the GA date (OSS lines: cloud builds them from source, so they have no artifact). -# OSS lines are tagged ee-vX.Y.Z(-rc.N); plain vX.Y.Z tags there are community releases from main, never a line's. +# On-prem support: 184 days (longest 6 months) from the line's first stable chart push, and only the newest 6 GA +# lines. +WINDOW = dt.timedelta(days=184) +WINDOW_LINES = 6 +# Per repo, so both repos run this file; use() loads an entry as (TARGETS, LATEST_ONLY: only the latest GA line is +# supported, KEPT: supported while the branch exists, PINS: repo whose tags pyproject.toml pins, PREFIX: tag prefix). +# TARGETS: target -> (build artifact, GA anchor artifact), each /; RELSTATE_AR_ names its +# repository. GIT: the line's own git tags stand in for an artifact (OSS lines). OSS lines are tagged +# ee-vX.Y.Z(-rc.N). GIT = "git" REPOS = { - # Cloud: each target's build artifact is backend, since every build pushes it (all services share one tag). + # backend: every build pushes it, and all services share one tag. "unstract-cloud": ( { "cloud": ("cloud/backend", "cloud/backend"), @@ -164,9 +160,7 @@ def registry_tags(artifact): def _read(artifact, registry): - """registry(artifact), with a chart push's X.Y.Z tag read as vX.Y.Z. A chart push is tagged with its Chart.yaml - version, which has no v; pushes before the release-line flow were tagged vX.Y.Z and read as they are. - """ + """registry(artifact), with chart tags X.Y.Z read as vX.Y.Z (charts are tagged without the v).""" tags = registry(artifact) if not artifact.startswith("chart/"): return tags @@ -181,15 +175,17 @@ def taken_tags(reg=None, registry=registry_tags): reg: {artifact: tags} already read; artifacts missing from it are read. """ reg = reg or {} - return { - t - for a in sorted({a for pair in TARGETS.values() for a in pair} - {GIT}) - for t in (reg[a] if a in reg else registry(a)) - } + return {t for a in _artifacts() for t in (reg[a] if a in reg else registry(a))} + +def _artifacts(git=False): + """Sorted registry artifacts of TARGETS; GIT too when git.""" + return sorted( + {a for pair in TARGETS.values() for a in pair} - (set() if git else {GIT}) + ) -def _ts(s): - return dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + +_ts = rm.ts def _key(line): @@ -230,10 +226,23 @@ def ahead_of_ga(repo, ref, line): def _tag_commit(repo, tag): - r = rm.git( - repo, "rev-parse", "-q", "--verify", f"refs/tags/{tag}^{{commit}}", check=False - ) - return r.stdout.strip() if r.returncode == 0 else None + return rm.resolve(repo, f"refs/tags/{tag}") + + +def line_tags(repo, line, *args, prefix=None): + """Version tags of line X.Y from `git tag -l`; prefix: the tag prefix, this repo's by default.""" + prefix = PREFIX if prefix is None else prefix + ver = ver_re(prefix) + return [ + t + for t in rm.git(repo, "tag", *args, "-l", f"{prefix}v{line}.*").split() + if (m := ver.fullmatch(t)) and f"{m[1]}.{m[2]}" == line + ] + + +def older_newer(a, b): + """(older, newer) of two release/X.Y branches.""" + return tuple(sorted((a, b), key=lambda n: _key(n.split("/", 1)[1]))) def moved_archive(repo, line, sha): @@ -273,7 +282,7 @@ def rc_slack(repo, tag): def target_state(t, name, rows, now): - """State of target t on an enabled branch. The one place the GA/supported convention lives. + """State of target t on an enabled branch. Single source of the GA/support rules. On-prem: stabilized for WINDOW after the line's first stable chart push (a patch does not extend it), and only while among the newest WINDOW_LINES lines with an on-prem GA. A LATEST_ONLY target (cloud): only the latest GA branch @@ -282,8 +291,7 @@ def target_state(t, name, rows, now): then stabilized while the branch exists; the cloud lines pinning it decide its retirement, and cloud Retire names it. """ ga = rows[name]["targets"][t]["ga"] - # Known limit: a target built but never GA (on-prem RCs with no chart push) stays stabilizing for good, so its - # line never reaches the retire list; a person retires it with Retire `retire_line`. + # Known limit: a target never GA stays stabilizing; retire it with Retire retire_line. if not ga: return "stabilizing" if t in KEPT: @@ -314,12 +322,8 @@ def _main(repo): def _next_minor(repo, ordered): - """One minor past the highest release/X.Y line, shipped retired line or stable vX.Y.Z on main, so a cut never - reuses a minor this flow shipped. Of the stable vX.Y.Z tags, only those merged into main count, so one off main (an - old-flow release not merged back, or a stray such as v100.0.x) does not. A line retired early before any GA (retired/X.Y-, - its RCs deleted) is reused. - - With no release/X.Y branch yet (cutover) only the tags count. + """One minor past the highest release line, shipped retired line or stable tag merged into main (a stable tag + off main, e.g. a release not merged back, does not count). A line retired before GA is reused. """ main = _main(repo) lines = [ @@ -336,113 +340,114 @@ def _next_minor(repo, ordered): return f"{max(lines)[0]}.{max(lines)[1] + 1}" if lines else None -def derive(repo, now, registry=registry_tags, rc=None): - t0 = time.monotonic() - reg = { - a: git_tags(repo) if a == GIT else _read(a, registry) - for a in sorted({a for pair in TARGETS.values() for a in pair}) +def _row(repo, name, ref, taken): + """The derive() row of one release/X.Y branch, before its targets are filled in.""" + # The tip's RC tag if built, else the next one. + tip = rm.rc_version(repo, ref, PREFIX, taken) + built = tip in rm.git(repo, "tag", "--points-at", ref).split() + line = name.split("/", 1)[1] + row = { + "branch": name, + "line": line, + "state": None, + "enabled": [], + "next_rc": None if built else tip, + "tip_rc": tip if built else None, + "ahead_of_ga": ahead_of_ga(repo, ref, line), + "targets": {}, } - print( - f"relstate: registry read {len(reg)} artifacts in {time.monotonic() - t0:.2f}s", - file=sys.stderr, - ) # registry read cost - refs = _branches(repo) - rows, anomalies = {}, [] - taken = taken_tags(reg) - for name, ref in refs.items(): - tip = rm.rc_version( - repo, ref, PREFIX, taken - ) # the tip's RC tag if built (one RC per commit), else the next one - built = tip in rm.git(repo, "tag", "--points-at", ref).split() - rows[name] = { - "branch": name, - "line": name.split("/", 1)[1], - "state": None, - "enabled": [], - "next_rc": None if built else tip, - "tip_rc": tip if built else None, - "ahead_of_ga": ahead_of_ga(repo, ref, name.split("/", 1)[1]), - "targets": {}, - } - if built and (url := rc_slack(repo, tip)): # only when the RC has a link - rows[name]["tip_rc_slack"] = url - # A line with no branch but a git stable tag was promoted and then retired: its registry tags are - # history, not anomalies, and are skipped before any per-tag git call. A line that never - # promoted (RC-only) stays an anomaly. Retire only lists lines that were GA, so this is exact. - shipped = {} + if built and (url := rc_slack(repo, tip)): # only when the RC has a link + row["tip_rc_slack"] = url + return row + + +def _anomaly_reason(repo, name, refs, sha): + """Why a build tag enables nothing on its line, or None when its commit is on the branch.""" + if name not in refs: + return "no branch" + if not sha: + return "no git tag" + if not rm.is_ancestor(repo, sha, refs[name]): + return "commit not on branch" + return None - # enabled: a build tag of T whose git commit is on the branch (derived, never recorded). - for t, (builds, anchor) in TARGETS.items(): - for tag in sorted(reg[builds]): - m = VER_RE.fullmatch(tag) - if not m: - continue # not a release version (e.g. smoke tags) - name = f"release/{m[1]}.{m[2]}" - if name not in refs: - if name not in shipped: # one git tag read per line, not per registry tag - shipped[name] = bool(rm._tags(repo, m[1], m[2], PREFIX)[0]) - if shipped[name]: - continue - sha = _tag_commit(repo, tag) - if name not in refs or not sha or not rm.is_ancestor(repo, sha, refs[name]): - why = ( - "no branch" - if name not in refs - else "no git tag" - if not sha - else "commit not on branch" - ) - anomalies.append( - { - "target": t, - "artifact": builds, - "tag": tag, - "branch": name, - "reason": why, - } - ) - continue - tt = rows[name]["targets"].setdefault( - t, {"state": None, "rcs": [], "ga": None, "promoted": None} + +def _shipped(repo, name, m, shipped): + """Whether the branchless line name (version match m) has a git stable tag, cached in shipped.""" + if name not in shipped: # one git tag read per line, not per registry tag + shipped[name] = bool(rm._tags(repo, m[1], m[2], PREFIX)[0]) + return shipped[name] + + +def _enable(repo, t, builds, reg, refs, rows, shipped, anomalies): + """Mark target t enabled on each branch holding one of its build tags; anomalies get the rest.""" + for tag in sorted(reg[builds]): + m = VER_RE.fullmatch(tag) + if not m: + continue # not a release version (e.g. smoke tags) + name = f"release/{m[1]}.{m[2]}" + if name not in refs and _shipped(repo, name, m, shipped): + continue + sha = _tag_commit(repo, tag) + if why := _anomaly_reason(repo, name, refs, sha): + anomalies.append( + { + "target": t, + "artifact": builds, + "tag": tag, + "branch": name, + "reason": why, + } ) - if m[4]: - tt["rcs"].append(tag) - for name, row in rows.items(): - tt = row["targets"].get(t) - if not tt: - continue - tt["rcs"].sort(key=_vkey) - # On-prem GA reads the chart artifact. Publish pushes the stable chart as X.Y.Z, read as vX.Y.Z (_read); - # RC charts are never pushed, and legacy 0.0.0-rc.N pushes match no line. - stable = re.compile(rf"{re.escape(PREFIX)}v{re.escape(row['line'])}\.\d+") - # ga (the anchor's first stable) anchors state and the support window; promoted (the build artifact's - # latest stable) is what ships next - for key, artifact, pick in (("ga", anchor, 0), ("promoted", builds, -1)): - if tags := sorted( - (g for g in reg[artifact] if stable.fullmatch(g)), key=_vkey - ): - tt[key] = {"tag": tags[pick], **reg[artifact][tags[pick]]} - if t in KEPT: - tt["stabilized"] = bool(_tag_commit(repo, STABILIZED + row["line"])) + continue + tt = rows[name]["targets"].setdefault( + t, {"state": None, "rcs": [], "ga": None, "promoted": None} + ) + if m[4]: + tt["rcs"].append(tag) + + +def _fill_ga(repo, t, builds, anchor, reg, rows): + """Sort target t's RCs and set its ga / promoted (and KEPT stabilized) on every branch it is enabled on.""" + for row in rows.values(): + tt = row["targets"].get(t) + if not tt: + continue + tt["rcs"].sort(key=_vkey) + # ga: the anchor's first stable (state, window). promoted: the build artifact's latest stable (ships next). + # Charts are read as vX.Y.Z (_read). + stable = re.compile(rf"{re.escape(PREFIX)}v{re.escape(row['line'])}\.\d+") + for key, artifact, pick in (("ga", anchor, 0), ("promoted", builds, -1)): + if tags := sorted( + (g for g in reg[artifact] if stable.fullmatch(g)), key=_vkey + ): + tt[key] = {"tag": tags[pick], **reg[artifact][tags[pick]]} + if t in KEPT: + tt["stabilized"] = bool(_tag_commit(repo, STABILIZED + row["line"])) + + +def _line_state(states): + """A line's state from its targets' states.""" + if not states or "stabilizing" in states: + return "stabilizing" + if "stabilized" in states: + return "stabilized" + return "past-eos" + +def _set_states(rows, now): + """Set every target's and line's state and enabled list on rows.""" for name, row in rows.items(): for t, tt in row["targets"].items(): tt["state"] = target_state(t, name, rows, now) row["enabled"] = sorted(row["targets"]) - states = {tt["state"] for tt in row["targets"].values()} # No enabled target (cut, nothing built yet) is stabilizing: every line builds cloud by default. - row["state"] = ( - "stabilizing" - if not states or "stabilizing" in states - else "stabilized" - if "stabilized" in states - else "past-eos" - ) + row["state"] = _line_state({tt["state"] for tt in row["targets"].values()}) - ordered = sorted(rows.values(), key=lambda r: _key(r["line"]), reverse=True) - out = { - "now": now.strftime("%Y-%m-%dT%H:%M:%SZ"), - "branches": ordered, + +def _sets(ordered): + """The supported, backport_targets and retire sets of derive() over the ordered rows.""" + return { "supported": { t: [ r["branch"] @@ -467,6 +472,38 @@ def derive(repo, now, registry=registry_tags, rc=None): for r in ordered if r["state"] == "past-eos" and any(tt["ga"] for tt in r["targets"].values()) ], + } + + +def derive(repo, now, registry=registry_tags, rc=None): + t0 = time.monotonic() + reg = { + a: git_tags(repo) if a == GIT else _read(a, registry) + for a in _artifacts(git=True) + } + print( + f"relstate: registry read {len(reg)} artifacts in {time.monotonic() - t0:.2f}s", + file=sys.stderr, + ) + refs = _branches(repo) + anomalies = [] + taken = taken_tags(reg) + rows = {name: _row(repo, name, ref, taken) for name, ref in refs.items()} + # A branchless line with a stable tag was retired after GA: its registry tags are history, not anomalies. + shipped = {} + + # enabled: a build tag of T whose git commit is on the branch (derived, never recorded). + for t, (builds, anchor) in TARGETS.items(): + _enable(repo, t, builds, reg, refs, rows, shipped, anomalies) + _fill_ga(repo, t, builds, anchor, reg, rows) + + _set_states(rows, now) + + ordered = sorted(rows.values(), key=lambda r: _key(r["line"]), reverse=True) + out = { + "now": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "branches": ordered, + **_sets(ordered), "next_minor": _next_minor(repo, ordered), "anomalies": anomalies, } @@ -480,7 +517,7 @@ def retire_line_refusal(state, branch, force=False): promoted, say) may go. A stabilized on-prem target (chart in its window) goes only with force. Stabilized cloud is production and never goes: relstate would make the next older cloud GA production again. """ - row = next((b for b in state["branches"] if b["branch"] == branch), None) + row = _branch_row(state, branch) if not row: return "no such branch" held = [t for t, tt in sorted(row["targets"].items()) if tt["state"] == "stabilized"] @@ -491,6 +528,43 @@ def retire_line_refusal(state, branch, force=False): return None +def _branch_row(state, branch): + """The row of branch in state, or None.""" + return next((b for b in state["branches"] if b["branch"] == branch), None) + + +def _by_digest(tags): + """{digest: {id, digest, tags}} of an artifact's {tag: info}.""" + by_digest = {} + # Version order, so the last version is the newest RC (rc.10 after rc.9); other tags first. + for t, info in sorted( + tags.items(), + key=lambda kv: (_vkey(kv[0]) if VER_RE.fullmatch(kv[0]) else (-1, -1), kv[0]), + ): + by_digest.setdefault( + info["digest"], + {"id": info.get("id"), "digest": info["digest"], "tags": []}, + )["tags"].append(t) + return by_digest + + +def _abandon_versions(reg, rc_re): + """abandon_plan's versions to delete per artifact and the last versions kept.""" + versions, kept = {}, [] + for a, tags in reg.items(): + by_digest = _by_digest(tags) + drop = [ + v for v in by_digest.values() if all(rc_re.fullmatch(t) for t in v["tags"]) + ] + # Known limit: untagged versions are not listed, so "last" counts tagged ones only. + if drop and len(drop) == len(by_digest): + kept.append(f"{a}:{','.join(drop[-1]['tags'])}") + drop = drop[:-1] + if drop: + versions[a] = drop + return versions, kept + + def abandon_plan(repo, branch, registry=registry_tags): """What Retire `abandon` deletes for an existing release/X.Y line that never shipped (cut by mistake, or built only as RCs). @@ -501,10 +575,7 @@ def abandon_plan(repo, branch, registry=registry_tags): """ x, y = rm._line(branch) rc_re = re.compile(rf"{re.escape(PREFIX)}v{x}\.{y}\.\d+-rc\.\d+") - reg = { - a: _read(a, registry) - for a in sorted({a for pair in TARGETS.values() for a in pair} - {GIT}) - } + reg = {a: _read(a, registry) for a in _artifacts()} main = _main(repo) ga = [f"{PREFIX}v{x}.{y}.{z}" for z in sorted(rm._tags(repo, x, y, PREFIX)[0])] ga += [ @@ -519,24 +590,7 @@ def abandon_plan(repo, branch, registry=registry_tags): if rm.classify_commit(repo, c, main=main) not in ("build-generated", "cherry-pick", "declined") ] - versions, kept = {}, [] - for a, tags in reg.items(): - by_digest = {} - # Known limit: string order, so the version kept as last below can be rc.9's when rc.10 exists. - for t, info in sorted(tags.items()): - by_digest.setdefault( - info["digest"], - {"id": info.get("id"), "digest": info["digest"], "tags": []}, - )["tags"].append(t) - drop = [ - v for v in by_digest.values() if all(rc_re.fullmatch(t) for t in v["tags"]) - ] - # ponytail: untagged versions are invisible here, so "last" is counted over tagged ones (conservative). - if drop and len(drop) == len(by_digest): - kept.append(f"{a}:{','.join(drop[-1]['tags'])}") - drop = drop[:-1] - if drop: - versions[a] = drop + versions, kept = _abandon_versions(reg, rc_re) rc_tags = sorted( ( t @@ -575,10 +629,8 @@ def promote_blockers(state, branch, target): An older line blocks only where `target` is enabled on it and it has no GA for `target`; GA is the cloud promote retag or the on-prem chart push. A retired line is gone from the state, so it never blocks. + Not applied to KEPT targets: each OSS line goes GA with its cloud line. """ - # Not applied to a KEPT target (OSS). The order protects customers who stay on a line and upgrade later - # (on-prem windows). Each OSS line goes GA with its cloud line, so the order would hold an OSS line until the - # previous cloud line is GA. if target in KEPT: return [] line = _key(branch.split("/", 1)[1]) @@ -598,7 +650,7 @@ def enable_blockers(state, branch, target): wait to go GA behind a newer GA line whose patches the promote order would then hold. A target already enabled on `branch` is not affected (a patch RC on an old line still builds). """ - row = next((b for b in state["branches"] if b["branch"] == branch), None) + row = _branch_row(state, branch) if row and target in row["targets"]: return [] line = _key(branch.split("/", 1)[1]) @@ -623,13 +675,17 @@ def cell(tt): return f"stabilizing (rc.{VER_RE.fullmatch(tt['rcs'][-1])[4]})" return tt["state"] + def slack(b): + u = b.get("tip_rc_slack") + return f" ([Slack]({u}))" if u else "" + out = [ f"| branch | state | {' | '.join(TARGETS)} | tip_rc | next_rc | ahead_of_ga |", "|---|---|" + "---|" * (len(TARGETS) + 3), ] out += [ f"| {b['branch']} | {b['state']} | {' | '.join(cell(b['targets'].get(t)) for t in TARGETS)} " - f"| {b['tip_rc'] or '-'}{f' ([Slack]({u}))' if (u := b.get('tip_rc_slack')) else ''} | {b['next_rc'] or '-'} | {'-' if b['ahead_of_ga'] is None else b['ahead_of_ga']} |" + f"| {b['tip_rc'] or '-'}{slack(b)} | {b['next_rc'] or '-'} | {'-' if b['ahead_of_ga'] is None else b['ahead_of_ga']} |" for b in state["branches"] ] return ( @@ -653,13 +709,15 @@ def main(argv=None, registry=registry_tags): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--repo", default=".") p.add_argument("--now", type=_ts, default=None, help="ISO-8601 UTC; default: now") - p.add_argument("--rc", help="also report the stable tag promoting this RC would get") + p.add_argument( + "--rc", + type=rm.option_free, + help="also report the stable tag promoting this RC would get", + ) a = p.parse_args(argv) try: state = derive(a.repo, a.now or dt.datetime.now(dt.UTC), registry, a.rc) - except ( - Exception - ) as e: # registry or git failure: no JSON, never a silently empty state + except Exception as e: # never print a silently empty state traceback.print_exc() print(f"relstate: {e}", file=sys.stderr) return 1 diff --git a/.github/release/test_backport.py b/.github/release/test_backport.py index 7435e5031e..775208b865 100644 --- a/.github/release/test_backport.py +++ b/.github/release/test_backport.py @@ -1,4 +1,4 @@ -"""backport helpers: held_filter and the held summary line; PR body of a pick PR; picks in main order; branches filter; accept= and UNCLASSIFIABLE; owed-picks summary; dispatch route for a non-fix; push summary (status rows, latest tag, stale mark, quiet line); push rescan plan; PR listing classes; push retry; stuck clean pick PR; open conflict PRs per line; pick against real git (no untracked file in a conflict commit, a path with a space, a pick merged after the fetch). Run: python test_backport.py""" +"""Tests for backport helpers, and pick() against real git. Run: python test_backport.py""" import io import json @@ -9,7 +9,7 @@ from unittest import mock os.environ["GITHUB_REPOSITORY"] = ( - "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets + "Zipstack/unstract-cloud" # pin relstate to the cloud config ) os.environ["RELEASE_BOT_APP_SLUG"] = "zipstack-push-to-main" # the release App BOT = { @@ -20,6 +20,10 @@ import relmgr as rm +def dev_git(repo, *a): + return rm.git(repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a) + + class Held(unittest.TestCase): def test_held_filter(self): a, b, c, d, e = (ch * 40 for ch in "abcde") @@ -145,9 +149,7 @@ def test_branches_narrow(self): class Listing(unittest.TestCase): def test_classes_and_fallback(self): with tempfile.TemporaryDirectory() as repo: - git = lambda *a: rm.git( - repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a - ) + git = lambda *a: dev_git(repo, *a) git("init", "-q", "-b", "main") shas = [] for i in range(6): # cut, fix, security, misc, bad title, no PR @@ -201,10 +203,13 @@ def test_classes_and_fallback(self): ) # an unmerged row (no merge commit) is skipped looked = [] saved = bp.fix_class - bp.fix_class = lambda slug, sha, token=None: looked.append(sha) or ( - None, - "no PR", - None, + bp.fix_class = lambda slug, sha, token=None: ( + looked.append(sha) + or ( + None, + "no PR", + None, + ) ) try: rows = bp.held("o/r", repo, "release/0.1", accept=bp.OWED, cache=classes) @@ -254,14 +259,14 @@ def run(*args, check=True, input=None): self.assertEqual((sleeps, len(calls)), ([5, 20], 3)) calls.clear(), sleeps.clear() bp.run = fake([1, 1, 1]) - with mock.patch.object(bp.rm, "is_converged", lambda *a, **k: False): + with mock.patch.object(bp.rm, "is_converged", return_value=False): out = bp.push_head("o/r", sha, branch, head, "#7", sleep=sleeps.append) comment = calls[-1] calls.clear(), sleeps.clear() bp.run = fake( [1, 1, 1] ) # rejected because another run landed the pick: no failure, no comment - with mock.patch.object(bp.rm, "is_converged", lambda *a, **k: True): + with mock.patch.object(bp.rm, "is_converged", return_value=True): landed = bp.push_head("o/r", sha, branch, head, "#7", sleep=sleeps.append) finally: bp.run = saved @@ -284,13 +289,13 @@ def run(*args, check=True, input=None): ) -ORIG = "77212a38cc07a0e7fc5f31c691275666976c4c82" +ORIG = "0123abc" + "d" * 33 class Body(unittest.TestCase): def test_conflict_body_has_no_stale_rule(self): body = bp.pr_body( - ORIG, "#57", "release/0.185", "backport/0.185/77212a3", ["NOTES.md"] + ORIG, "#5", "release/1.6", f"backport/1.6/{ORIG[:7]}", ["NOTES.md"] ) self.assertNotIn("do not merge by hand", body) self.assertIn( @@ -306,7 +311,7 @@ def test_conflict_body_has_no_stale_rule(self): ) self.assertIn( "Clean pick", - bp.pr_body(ORIG, "#57", "release/0.185", "backport/0.185/77212a3"), + bp.pr_body(ORIG, "#5", "release/1.6", f"backport/1.6/{ORIG[:7]}"), ) @@ -318,18 +323,7 @@ def test_newest_first_comes_back_oldest_first(self): for i, date in enumerate(("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z")): os.environ["GIT_COMMITTER_DATE"] = date try: - rm.git( - repo, - "-c", - "user.name=Dev", - "-c", - "user.email=dev@example.com", - "commit", - "-q", - "--allow-empty", - "-m", - f"fix {i}", - ) + dev_git(repo, "commit", "-q", "--allow-empty", "-m", f"fix {i}") finally: del os.environ["GIT_COMMITTER_DATE"] shas.append(rm.git(repo, "rev-parse", "HEAD")) @@ -481,18 +475,7 @@ def test_next_step_first_match_wins(self): def test_latest_tag_counts_since_any_tag(self): with tempfile.TemporaryDirectory() as repo: rm.git(repo, "init", "-q", "-b", "release/0.1") - commit = lambda m: rm.git( - repo, - "-c", - "user.name=Dev", - "-c", - "user.email=dev@example.com", - "commit", - "-q", - "--allow-empty", - "-m", - m, - ) + commit = lambda m: dev_git(repo, "commit", "-q", "--allow-empty", "-m", m) commit("cut") rm.git(repo, "tag", "v0.1.0-rc.1") rm.git(repo, "tag", "v0.1.0") # promoted: same commit, the stable tag wins @@ -709,10 +692,10 @@ def finish(self, labels=(), oid=None, prs=None, author=BOT): squash = lambda slug, pr, msg: self.merged.append((pr["number"], msg)) or True with ( mock.patch.object( - bp, "gh", lambda *a: json.dumps(listing if prs is None else prs) + bp, "gh", return_value=json.dumps(listing if prs is None else prs) ), mock.patch.object(bp.bm, "squash", squash), - mock.patch.object(bp, "soft", lambda *a: None), + mock.patch.object(bp, "soft", return_value=None), ): return bp.finish_open("o/r", "release/1.0", "backport/1.0/aaaaaaa") @@ -895,13 +878,12 @@ def setUp(self): ["git", "clone", "-q", d + "/o", d + "/w"], check=True, capture_output=True ) self.write(d + "/w/a.txt", "x\n") - ( - g("add", "a.txt"), - g("commit", "-q", "-m", "base"), - g("push", "-q", "origin", "HEAD:main", "HEAD:release/1.0"), - ) + g("add", "a.txt") + g("commit", "-q", "-m", "base") + g("push", "-q", "origin", "HEAD:main", "HEAD:release/1.0") self.write(d + "/w/a.txt", "y\n") - g("commit", "-q", "-am", "fix: y (#5)"), g("push", "-q", "origin", "HEAD:main") + g("commit", "-q", "-am", "fix: y (#5)") + g("push", "-q", "origin", "HEAD:main") self.fix = g("rev-parse", "HEAD") g("fetch", "-q", "origin") self.d, cwd = d, os.getcwd() @@ -923,8 +905,8 @@ def gh(self, *a, token=None): def pick(self): with ( mock.patch.object(bp, "gh", self.gh), - mock.patch.object(bp.bm, "squash", lambda *a: True), - mock.patch.object(bp, "soft", lambda *a: None), + mock.patch.object(bp.bm, "squash", return_value=True), + mock.patch.object(bp, "soft", return_value=None), ): return bp.pick("o/r", self.fix, "release/1.0", "UN-1 [FIX] y", "#5") @@ -939,11 +921,9 @@ def test_clean_pick_carries_fix_labels(self): def test_conflict_pick_carries_fix_labels(self): self.g("checkout", "-q", "-B", "rel", "origin/release/1.0") self.write(self.d + "/w/a.txt", "z\n") - ( - self.g("commit", "-q", "-am", "z"), - self.g("push", "-q", "origin", "HEAD:release/1.0"), - self.g("fetch", "-q", "origin"), - ) + self.g("commit", "-q", "-am", "z") + self.g("push", "-q", "origin", "HEAD:release/1.0") + self.g("fetch", "-q", "origin") self.assertEqual(self.pick()[0], "pr") create = next(a for a in self.calls if a[:2] == ("pr", "create")) self.assertEqual( @@ -955,11 +935,10 @@ def test_empty_pick_names_the_decline(self): self.g("checkout", "-q", "-B", "rel", "origin/release/1.0") self.write(self.d + "/w/a.txt", "y\n") self.write(self.d + "/w/b.txt", "b\n") - self.g("add", "."), self.g("commit", "-q", "-m", "y and b, by hand") - ( - self.g("push", "-q", "origin", "HEAD:release/1.0"), - self.g("fetch", "-q", "origin"), - ) + self.g("add", ".") + self.g("commit", "-q", "-m", "y and b, by hand") + self.g("push", "-q", "origin", "HEAD:release/1.0") + self.g("fetch", "-q", "origin") out = self.pick() self.assertEqual(out[0], "failed") self.assertIn(f"pick of {self.fix[:7]} onto release/1.0 is empty", out[1]) @@ -986,7 +965,7 @@ def test_relevance_failure_never_fails_the_pick(self): mock.patch.object( bp.rm, "relevance", side_effect=RuntimeError("bad | path\nsecond line") ), - mock.patch.object(bp, "pick", lambda *a: ("merged", "abc1234 (#8)")), + mock.patch.object(bp, "pick", return_value=("merged", "abc1234 (#8)")), mock.patch("sys.stdout", io.StringIO()), ): row = bp.pick_row("o/r", "c" * 40, "release/1.0", "fix", "#5", "t") @@ -995,7 +974,7 @@ def test_relevance_failure_never_fails_the_pick(self): def test_failure_names_its_type(self): with ( - mock.patch.object(bp.rm, "relevance", lambda *a: "relevant"), + mock.patch.object(bp.rm, "relevance", return_value="relevant"), mock.patch.object(bp, "pick", side_effect=KeyError("x")), mock.patch.object(bp, "run"), mock.patch("sys.stdout", io.StringIO()), @@ -1042,16 +1021,18 @@ def commit(msg, **files): for name, text in files.items(): with open(os.path.join(d, name.replace("_", " ")), "w") as f: f.write(text) - g("add", "-A"), g("commit", "-q", "-m", msg) + g("add", "-A") + g("commit", "-q", "-m", msg) return g("rev-parse", "HEAD") g("init", "-q", "-b", "main") commit("base", a="1\n2\n", b="1\n", c_d="1\n") g("branch", "release/1.0") main = commit("rewrite a:2 after the cut", a="1\nTWO\n", b="1\nnew\n") - rel = lambda **files: g( - "checkout", "-q", "-B", "fix", main - ) or bp.rm.relevance(d, commit("fix", **files), "release/1.0") + rel = lambda **files: ( + g("checkout", "-q", "-B", "fix", main) + or bp.rm.relevance(d, commit("fix", **files), "release/1.0") + ) self.assertEqual(rel(a="ONE\nTWO\n"), "relevant") # a:1 is as on the branch self.assertEqual( rel(a="ONE\nTWO!\n"), "partial" @@ -1162,5 +1143,17 @@ class PickGitPathWithSpace(PickGit): PATH = "docker (deprecated)/é.txt" +class Guards(unittest.TestCase): + def test_resolve_refuses_an_option(self): + with mock.patch.object(bp.rm, "resolve") as resolve: + self.assertIsNone(bp.resolve("o/r", "#--all")) + resolve.assert_not_called() + + def test_main_refuses_a_state_file_outside(self): + argv = ["backport.py", "--state", "/nonexistent/state.json", "--rescan"] + with mock.patch("sys.argv", argv), self.assertRaises(ValueError): + bp.main() + + if __name__ == "__main__": unittest.main() diff --git a/.github/release/test_backport_merge.py b/.github/release/test_backport_merge.py index f995c6484f..c797d41095 100644 --- a/.github/release/test_backport_merge.py +++ b/.github/release/test_backport_merge.py @@ -1,6 +1,4 @@ -"""Pure-logic checks for backport_merge (decline reason, landed, merge, close-is-decline, re-runs, reopen, squash retry), with gh -and git injected. Run: python test_backport_merge.py -""" +"""Tests for backport_merge with gh and git stubbed. Run: python test_backport_merge.py""" import json import os @@ -158,7 +156,7 @@ def setUp(self): def close(self, *a, **k): bm.gh = Gh(*a, **k) - return bm.close("o/r", 7, ME) + return bm.record("o/r", 7, ME)["result"] def test_merged(self): out = self.close("MERGED", labels=["backport-conflict", "backport-declined"]) @@ -294,7 +292,9 @@ def test_head_without_a_pick_declines_nothing(self): for ref in (None, "d" * 40): bm.gh = Gh("CLOSED") bm.gh.head_ref = ref - self.assertEqual(bm.close("o/r", 7, ME), "skipped: no pick, head gone") + self.assertEqual( + bm.record("o/r", 7, ME)["result"], "skipped: no pick, head gone" + ) self.assertEqual(bm.gh.writes(), [("pr", "comment")]) said = bm.gh.said()[0] self.assertIn("no longer holds this PR's head", said) @@ -305,7 +305,7 @@ def test_head_without_a_pick_declines_nothing(self): "gh api repos/o/r/git/ref/heads/... (exit 1): gh: Bad credentials (HTTP 401)" ) with self.assertRaisesRegex(RuntimeError, "HTTP 401"): - bm.close("o/r", 7, ME) + bm.record("o/r", 7, ME)["result"] self.assertEqual(bm.gh.writes(), []) def test_reopened_sets_conflict_only(self): @@ -332,14 +332,14 @@ def test_not_a_backport_pr_refused(self): bm.gh = Gh("CLOSED", labels=["backport-conflict"]) bm.gh.pr.update(change) with self.assertRaises(RuntimeError): - bm.close("o/r", 7, ME) + bm.record("o/r", 7, ME)["result"] self.assertEqual(bm.gh.writes(), []) def test_unset_app_slug_refused(self): bm.gh = Gh("CLOSED", labels=["backport-conflict"]) with mock.patch.dict(os.environ, {"RELEASE_BOT_APP_SLUG": ""}): with self.assertRaisesRegex(RuntimeError, "RELEASE_BOT_APP_SLUG is unset"): - bm.close("o/r", 7, ME) + bm.record("o/r", 7, ME)["result"] self.assertEqual(bm.gh.writes(), []) def test_head_already_gone(self): @@ -352,7 +352,7 @@ def gh(*args, token=None): fake = Gh("MERGED", labels=["backport-merged"]) bm.gh = gh - self.assertEqual(bm.close("o/r", 7, ME), "merged") + self.assertEqual(bm.record("o/r", 7, ME)["result"], "merged") class FirstPick(unittest.TestCase): @@ -388,7 +388,7 @@ def test_message_from_main(self): self.addCleanup(os.chdir, cwd) pr = {"headRefName": f"backport/0.187/{o[:7]}"} edited = f"fix: edited by hand\n\n(cherry picked from commit {o[:12]})" - with mock.patch.object(bm, "head_messages", lambda pr: [edited]): + with mock.patch.object(bm, "head_messages", return_value=[edited]): self.assertEqual( bm.first_pick(pr), (f"fix: real (#9)\n\n(cherry picked from commit {o})", [o]), @@ -396,12 +396,12 @@ def test_message_from_main(self): with mock.patch.object( bm, "head_messages", - lambda pr: ["fix: x\n\n(cherry picked from commit " + "d" * 40 + ")"], + return_value=["fix: x\n\n(cherry picked from commit " + "d" * 40 + ")"], ): self.assertEqual( bm.first_pick({"headRefName": "backport/0.187/ddddddd"}), ("", []) ) # not on main - with mock.patch.object(bm, "head_messages", lambda pr: []): + with mock.patch.object(bm, "head_messages", return_value=[]): self.assertEqual(bm.first_pick(pr), (None, [])) # no commit past the base @@ -491,8 +491,9 @@ def test_pending_check_is_not_retried(self): ) # GitHub's words reach the log def test_other_error_is_not_retried(self): + calls = [RuntimeError("Head branch was modified (HTTP 409)"), None] with self.assertRaisesRegex(RuntimeError, "HTTP 409"): - self.run_squash([RuntimeError("Head branch was modified (HTTP 409)"), None]) + self.run_squash(calls) if __name__ == "__main__": diff --git a/.github/release/test_convgate.py b/.github/release/test_convgate.py index de0351cd68..cf4a84d64c 100644 --- a/.github/release/test_convgate.py +++ b/.github/release/test_convgate.py @@ -1,4 +1,4 @@ -"""Synthetic-repo tests for convgate (2b-2d revert of a pick = decline; 2g-2h DECLINED rows and decline_records; 5 a plain fix is owed to a stabilized line; 7a-7e MISSING messages; 7f-7g a promoted stable tag is read before its GA; 9a-9h checks table, release-branch-only window, count line, next steps; 10a-10d OSS pins). Run: python test_convgate.py""" +"""Synthetic-repo tests for convgate. Run: python test_convgate.py""" import contextlib import datetime as dt @@ -10,7 +10,7 @@ from unittest import mock os.environ["GITHUB_REPOSITORY"] = ( - "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets + "Zipstack/unstract-cloud" # pin relstate to the cloud config ) import convgate as cg import relmgr as rm @@ -21,10 +21,11 @@ os.environ.pop("GITHUB_STEP_SUMMARY", None) os.environ.pop("OSS_REPO", None) NOW = dt.datetime(2026, 9, 29, tzinfo=dt.UTC) +DEV = ["-c", "user.name=Dev", "-c", "user.email=dev@example.com"] class Gate(unittest.TestCase): - """Row 1 base: 1.5 and 1.6 cut, then C lands on main and is picked to 1.5 only; cloud RCs on both.""" + """Base: 1.5 and 1.6 cut, then C lands on main and is picked to 1.5 only; cloud RCs on both.""" def setUp(self): self._tmp = tempfile.TemporaryDirectory() @@ -41,9 +42,7 @@ def setUp(self): self.push("cloud/backend", "v1.6.0-rc.1", "release/1.6") def g(self, *args): - return rm.git( - self.repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args - ) + return rm.git(self.repo, *DEV, *args) def change(self, msg, branch): if branch: @@ -118,7 +117,7 @@ def run_gate( self.printed = buf.getvalue() return code, "\n".join(out) - def test_1_both_directions(self): + def test_both_directions(self): code, out = self.run_gate("release/1.5") self.assertEqual(code, 2) self.assertIn(", 1 missing,", out) @@ -126,7 +125,7 @@ def test_1_both_directions(self): self.assertEqual(code, 2) self.assertIn("**release/1.5** (older", out) - def test_2_decline_converges(self): + def test_decline_converges(self): self.decline(self.c, "release/1.6") for b in ("release/1.5", "release/1.6"): code, out = self.run_gate(b) @@ -135,7 +134,7 @@ def test_2_decline_converges(self): "1 picks, 0 in the RC, 1 declined on release/1.6, 1 unclassifiable", out ) - def test_2b_revert_of_pick_declines_on_newer(self): + def test_revert_of_pick_declines_on_newer(self): self.pick(self.c, "release/1.6") r = self.revert(self.tip("release/1.6"), "release/1.6") self.assertEqual(rm.classify_commit(self.repo, r), "declined") @@ -147,7 +146,7 @@ def test_2b_revert_of_pick_declines_on_newer(self): "1 picks, 0 in the RC, 1 declined on release/1.6, 1 unclassifiable", out ) - def test_2c_revert_of_pick_on_older_owes_nothing(self): + def test_revert_of_pick_on_older_owes_nothing(self): r = self.revert(self.tip("release/1.5"), "release/1.5", short=True) self.assertEqual(rm.classify_commit(self.repo, r), "declined") rows = rm.convergence_report(self.repo, "release/1.5", "release/1.6") @@ -158,7 +157,7 @@ def test_2c_revert_of_pick_on_older_owes_nothing(self): code, out = self.run_gate(b) self.assertEqual(code, 0, out) - def test_2e_pick_after_decline_ships(self): + def test_pick_after_decline_ships(self): """A decline undone by merging the reopened PR: the newest trailer wins, the fix counts as picked.""" self.decline(self.c, "release/1.6") self.pick(self.c, "release/1.6") @@ -171,7 +170,7 @@ def test_2e_pick_after_decline_ships(self): self.assertEqual(code, 0, out) self.assertIn("1 picks, 1 in the RC, 1 unclassifiable", out) - def test_2e2_pick_after_short_sha_decline_ships(self): + def test_pick_after_short_sha_decline_ships(self): """A hand-written decline may name the original by a short sha; the bot's later pick names it in full. Both are one original, so the pick still wins. """ @@ -185,7 +184,7 @@ def test_2e2_pick_after_short_sha_decline_ships(self): self.assertEqual(code, 0, out) self.assertIn("1 picks, 1 in the RC, 1 unclassifiable", out) - def test_2f_pick_after_decline_past_ga_still_owed(self): + def test_pick_after_decline_past_ga_still_owed(self): """Newer is GA at a tag without the fix; its decline was undone by a later pick: older's promote waits for newer's next patch instead of reading the stale decline as converged. """ @@ -197,7 +196,7 @@ def test_2f_pick_after_decline_past_ga_still_owed(self): self.assertEqual(code, 2, out) self.assertIn("into `refs/tags/v1.6.0`", out) - def test_2g_declined_row(self): + def test_declined_row(self): """A decline converges (exit 0) and its row names the reason and the decline commit.""" self.g("checkout", "-q", "release/1.6") self.g( @@ -220,7 +219,7 @@ def test_2g_declined_row(self): out, ) - def test_2g2_declined_on_older(self): + def test_declined_on_older(self): """A decline on the older line owes the newer line nothing: no row, nothing counted.""" self.g("checkout", "-q", "release/1.5") self.g( @@ -238,7 +237,7 @@ def test_2g2_declined_on_older(self): self.assertNotIn(d[:12], out) self.assertIn("0 in the RC, 1 unclassifiable\n", out) - def test_2h_decline_records(self): + def test_decline_records(self): """Empty decline: its last `Declined:` line (subject without one); revert of a pick: the revert's subject; a later pick clears the record. """ @@ -269,13 +268,14 @@ def test_2h_decline_records(self): rm.declined_originals(self.repo, "release/1.6"), ) - def test_2d_revert_of_non_pick_is_invalid(self): + def test_revert_of_non_pick_is_invalid(self): work = self.tip("release/1.6") # the fixture's "work on 1.6": not a pick r = self.revert(work, "release/1.6") self.assertEqual(rm.classify_commit(self.repo, r), "invalid") - self.assertIn(r, rm.check_branch(self.repo, "release/1.6")) + rows, _ = rm.ci_commits(self.repo, "release/1.6", "push", main="main") + self.assertIn(r, [c for c, kind in rows if kind == "invalid"]) - def test_8a_backport_pr_judged_as_its_squash(self): + def test_backport_pr_judged_as_its_squash(self): """A plain resolution commit after the pick passes on a backport/* PR; any other PR still classifies all.""" self.g("checkout", "-q", "-b", "backport/1.6/abcdef0", "release/1.6") self.g("cherry-pick", "-x", self.c) @@ -288,7 +288,7 @@ def test_8a_backport_pr_judged_as_its_squash(self): self.assertEqual(len(both), 2) self.assertEqual(rm.classify_commit(self.repo, both[1]), "invalid") - def test_8b_conflict_markers(self): + def test_conflict_markers(self): self.assertEqual(rm.conflict_markers(self.repo), []) self.g("checkout", "-q", "release/1.6") with open(os.path.join(self.repo, "f.txt"), "w") as f: @@ -299,7 +299,7 @@ def test_8b_conflict_markers(self): self.g("commit", "-q", "-m", "markers") self.assertEqual(rm.conflict_markers(self.repo), ["f.txt"]) - def test_7a_older_promote_names_conflict_pr(self): + def test_older_promote_names_conflict_pr(self): c7 = self.c[:7] code, out = self.run_gate( "release/1.5", annotate="error", prs={f"backport/1.6/{c7}": (12, "u")} @@ -312,7 +312,7 @@ def test_7a_older_promote_names_conflict_pr(self): self.assertIn(f"::error::{msg}", self.printed) self.assertIn(msg, out) - def test_7b_newer_promote_conflict_pr_on_itself(self): + def test_newer_promote_conflict_pr_on_itself(self): c7 = self.c[:7] code, _ = self.run_gate( "release/1.6", annotate="error", prs={f"backport/1.6/{c7}": (12, "u")} @@ -324,7 +324,7 @@ def test_7b_newer_promote_conflict_pr_on_itself(self): self.printed, ) - def test_7c_no_pr_fallback_warns(self): + def test_no_pr_fallback_warns(self): d = self.change("D: another fix (#9)", "main") self.decline(self.c, "release/1.6") # leave D as the only MISSING row self.pick(d, "release/1.5") @@ -351,7 +351,7 @@ def test_7c_no_pr_fallback_warns(self): ) self.assertIn(msg, out) - def test_7d_on_newer_tip_not_shipped(self): + def test_on_newer_tip_not_shipped(self): self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"][ "v1.6.0-rc.1" ] # 1.6 GA at v1.6.0 @@ -365,7 +365,7 @@ def test_7d_on_newer_tip_not_shipped(self): self.printed, ) - def test_7e_on_promoted_tip_not_in_its_rc(self): + def test_on_promoted_tip_not_in_its_rc(self): self.pick(self.c, "release/1.6") # on 1.6's tip after v1.6.0-rc.1 was cut code, _ = self.run_gate( "release/1.6", annotate="error", promote_ref="v1.6.0-rc.1" @@ -386,7 +386,7 @@ def onprem(self): self.g("tag", "v1.6.0", "v1.6.0-rc.1") self.reg["on-prem/backend"]["v1.6.0"] = self.reg["on-prem/backend"]["v1.6.0-rc.1"] - def test_7f_promoted_not_published_read_at_its_tag(self): + def test_promoted_not_published_read_at_its_tag(self): self.onprem() self.pick( self.c, "release/1.6" @@ -403,7 +403,7 @@ def test_7f_promoted_not_published_read_at_its_tag(self): self.printed, ) - def test_7g_patch_promoted_read_over_published(self): + def test_patch_promoted_read_over_published(self): self.onprem() self.reg["chart/unstract-platform"] = { "v1.6.0": self.reg["on-prem/backend"]["v1.6.0"] @@ -430,7 +430,7 @@ def checks(self, out, order=(), split=True): self.assertIn(out.splitlines()[0], detail) return table - def test_9a_checks_clean(self): + def test_checks_clean(self): self.decline(self.c, "release/1.6") _, out = self.run_gate("release/1.5") self.assertEqual( @@ -448,7 +448,7 @@ def test_9a_checks_clean(self): "1 unclassifiable | - |", ) - def test_9b_checks_missing_with_conflict_pr(self): + def test_checks_missing_with_conflict_pr(self): _, out = self.run_gate( "release/1.6", prs={f"backport/1.6/{self.c[:7]}": (12, "u")} ) @@ -493,15 +493,15 @@ def rbo( "commit", "-q", "-m", - f"{title}\n\n* Release-branch-only: UN-37\n\nRelease-branch-only: UN-37", + f"{title}\n\n* Release-branch-only: UN-1\n\nRelease-branch-only: UN-1", ) return self.tip(branch) - def test_9c_checks_rbo_absent(self): + def test_checks_rbo_absent(self): self.decline(self.c, "release/1.6") rbo = self.rbo("pin the base image") # its trailer twice: one ticket _, out = self.run_gate("release/1.5") - what = f"UN-37 pin the base image ({rm.commit_link(rbo)})" + what = f"UN-1 pin the base image ({rm.commit_link(rbo)})" self.assertIn(f"- `{rbo[:12]}` RBO-ABSENT {what}", out) self.assertNotIn("RBO-ABSENT original", out) self.assertEqual( @@ -510,14 +510,14 @@ def test_9c_checks_rbo_absent(self): "`release/1.6` | check whether the newer line needs its own equivalent |", ) # a title that already starts with the ticket names it once - rbo = self.rbo("UN-37 [MISC] pin the base image") + rbo = self.rbo("UN-1 [MISC] pin the base image") _, out = self.run_gate("release/1.5") self.assertIn( - f"- `{rbo[:12]}` RBO-ABSENT UN-37 [MISC] pin the base image ({rm.commit_link(rbo)})\n", + f"- `{rbo[:12]}` RBO-ABSENT UN-1 [MISC] pin the base image ({rm.commit_link(rbo)})\n", out, ) - def test_9e_rbo_window(self): + def test_rbo_window(self): self.decline(self.c, "release/1.6") old = self.rbo( "before the cut", "2020-01-01T00:00:00Z" @@ -541,7 +541,7 @@ def test_9e_rbo_window(self): self.assertIn(later[:12], out) self.assertNotIn(new[:12], out) - def test_9f_checks_name_lines_as_needed(self): + def test_checks_name_lines_as_needed(self): self.reg["cloud/backend"]["v1.5.0"] = self.reg["cloud/backend"][ "v1.5.0-rc.1" ] # 1.5 GA: stabilized @@ -559,7 +559,7 @@ def test_9f_checks_name_lines_as_needed(self): "| ✅ fixes match older lines | `release/1.5` 1 picks,", self.checks(out)[3] ) - def test_9g_count_line(self): + def test_count_line(self): self.pick(self.c, "release/1.6") _, out = self.run_gate("release/1.5") self.assertIn( @@ -599,7 +599,7 @@ def test_9g_count_line(self): out, ) - def test_9h_next_steps(self): + def test_next_steps(self): """The do-this items in table order, each once; the table cells are built from the same list.""" _, out = self.run_gate( "release/1.6", prs={f"backport/1.6/{self.c[:7]}": (12, "u")} @@ -627,7 +627,7 @@ def test_9h_next_steps(self): table[4].endswith("| promote or retire `release/1.5` |"), table[4] ) - def test_9d_main_blocks_on_promote_order(self): + def test_main_blocks_on_promote_order(self): self.decline( self.c, "release/1.6" ) # no MISSING row: only 1.5, not GA yet, holds 1.6 back @@ -661,14 +661,14 @@ def test_9d_main_blocks_on_promote_order(self): "| ⛔ promote order | release/1.5 is not GA for cloud yet |", f.read() ) - def test_3_scope_is_the_target(self): + def test_scope_is_the_target(self): self.push("on-prem/backend", "v1.5.0-rc.2", "release/1.5") code, out = self.run_gate("release/1.5", "on-prem") self.assertEqual(code, 0) self.assertIn("scanned branches: none", out) self.assertEqual(self.run_gate("release/1.5", "cloud")[0], 2) - def test_4_ga_branch_scanned_at_its_ga_tag(self): + def test_ga_branch_scanned_at_its_ga_tag(self): self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"][ "v1.6.0-rc.1" ] # promote, same digest @@ -681,7 +681,7 @@ def test_4_ga_branch_scanned_at_its_ga_tag(self): code, out = self.run_gate("release/1.5") self.assertEqual(code, 0, out) - def test_5_plain_fix_owed_to_stabilized(self): + def test_plain_fix_owed_to_stabilized(self): self.reg["cloud/backend"]["v1.6.0"] = self.reg["cloud/backend"]["v1.6.0-rc.1"] self.g("tag", "v1.6.0", "v1.6.0-rc.1") # 1.6 GA = stabilized for cloud code, out = self.run_gate("release/1.5") @@ -690,7 +690,7 @@ def test_5_plain_fix_owed_to_stabilized(self): f"- `{self.tip('release/1.5')[:12]}` MISSING original {self.c[:12]}", out ) - def test_6_shipped_prs(self): + def test_shipped_prs(self): """shipped_prs: a pick counts, a decline and a main-only commit do not; C has no (#n), so lookup decides.""" d, e = ( self.change("D: picked (#7)", "main"), @@ -714,7 +714,7 @@ def test_6_shipped_prs(self): rm.shipped_prs(self.repo, ["v1.5.1"]) self.assertEqual(rm.shipped_prs(self.repo, []), set()) - def test_7_shipped_prs_newest_trailer(self): + def test_shipped_prs_newest_trailer(self): """shipped_prs: newest trailer wins. A pick backed out by a later decline or revert is not shipped; a pick after a decline is. """ @@ -741,7 +741,7 @@ def test_7_shipped_prs_newest_trailer(self): {5, 23}, ) - def test_7b_shipped_prs_line_picks(self): + def test_shipped_prs_line_picks(self): """shipped_prs(line=): the new line's picks of shipped originals drop too, by lookup (clean pick, original's subject) or by their own (#n) (conflict squash); a pick of an original declined on the previous line, a release-branch-only commit and line=None do not change. @@ -764,7 +764,7 @@ def test_7b_shipped_prs_line_picks(self): "-q", "--amend", "-m", - f"UN-9 [FIX] F (#41)\n\n(cherry picked from commit {f[:12]})", + f"UN-1 [FIX] F (#41)\n\n(cherry picked from commit {f[:12]})", ) self.pick(e, "release/1.6") pe = self.tip("release/1.6") @@ -784,6 +784,39 @@ def test_7b_shipped_prs_line_picks(self): {5, 21, 23, 41}, ) # no lookup hit for the clean pick: its (#21) only, nothing raised + def test_main_refuses_paths_outside_and_an_option_as_branch(self): + args = ["--repo", self.repo, "--target", "cloud"] + with ( + contextlib.redirect_stdout(io.StringIO()) as out, + mock.patch.object(rs, "PINS", None), + ): + code = cg.main( + args + ["--branch", "release/1.6", "--state", "/nonexistent/s.json"] + ) + self.assertEqual(code, 1) + self.assertIn("::error::convgate: Refused", out.getvalue()) + state = os.path.join(self.repo, "state.json") + with open(state, "w") as f: + json.dump(rs.derive(self.repo, NOW, lambda a: self.reg.get(a, {})), f) + with ( + contextlib.redirect_stdout(io.StringIO()) as out, + mock.patch.object(rs, "PINS", None), + ): + code = cg.main( + args + + ["--branch", "release/1.6", "--state", state] + + ["--out", "/nonexistent/out.md"] + ) + self.assertEqual(code, 1) + self.assertIn("::error::convgate: Refused", out.getvalue()) + for opt in ("--branch=-x", "--promote-ref=-x"): + with ( + self.subTest(opt=opt), + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit), + ): + cg.main(args + ["--branch", "release/1.6", opt]) + class OssPins(unittest.TestCase): """Cloud 1.5 pins OSS ee-v1.5.1, which carries OSS fix F; cloud 1.6 pins ee-v1.6.0, cut before F.""" @@ -814,9 +847,7 @@ def setUp(self): self.repin(line, pin, f"v{line}.0-rc.1") def g(self, repo, *args): - return rm.git( - repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args - ) + return rm.git(repo, *DEV, *args) def commit(self, repo, msg, file="x.txt", text=None, bot=False): with open(os.path.join(repo, file), "w") as f: @@ -825,7 +856,7 @@ def commit(self, repo, msg, file="x.txt", text=None, bot=False): who = ["-c", "user.name=bot", "-c", f"user.email={rm.BOT_EMAIL}"] if bot else [] rm.git( repo, - *(who or ["-c", "user.name=Dev", "-c", "user.email=dev@example.com"]), + *(who or DEV), "commit", "-q", "-m", @@ -871,7 +902,7 @@ def run_gate(self, branch): ) return code, "\n".join(out), lines - def test_10a_missing_oss_fix_names_the_oss_line_then_the_rc(self): + def test_missing_oss_fix_names_the_oss_line_then_the_rc(self): code, out, lines = self.run_gate("release/1.6") self.assertEqual(code, 2) self.assertIn( @@ -904,7 +935,7 @@ def test_10a_missing_oss_fix_names_the_oss_line_then_the_rc(self): self.assertEqual(code, 2) self.assertIn(": take a new RC of release/1.6 to pin OSS release/1.6's tip.", out) - def test_10b_repinned_newer_line_converges(self): + def test_repinned_newer_line_converges(self): self.g(self.oss, "checkout", "-q", "release/1.6") self.g(self.oss, "cherry-pick", "-x", self.f) self.g(self.oss, "tag", "ee-v1.6.1") @@ -922,7 +953,7 @@ def test_10b_repinned_newer_line_converges(self): ["0 OSS picks, 0 in ee-v1.5.1"], ) - def test_10d_decline_on_the_oss_line_converges(self): + def test_decline_on_the_oss_line_converges(self): # Declined on OSS release/1.6 after ee-v1.6.0: the line owes nothing, though the pinned tag predates the decline. self.g(self.oss, "checkout", "-q", "release/1.6") self.g( @@ -938,7 +969,7 @@ def test_10d_decline_on_the_oss_line_converges(self): code, out, _ = self.run_gate("release/1.6") self.assertEqual(code, 0, out) - def test_10c_pinning_repo_needs_the_oss_clone(self): + def test_pinning_repo_needs_the_oss_clone(self): with ( contextlib.redirect_stdout(io.StringIO()) as buf, contextlib.redirect_stderr(io.StringIO()) as err, diff --git a/.github/release/test_links.py b/.github/release/test_links.py index 1707e6edde..b47c2cec0f 100644 --- a/.github/release/test_links.py +++ b/.github/release/test_links.py @@ -22,7 +22,6 @@ def test_actions(self): rm.tag_link("v0.192.0-rc.1"), f"[`v0.192.0-rc.1`]({WEB}/releases/tag/v0.192.0-rc.1)", ) - self.assertEqual(rm.run_link(9), f"[run]({WEB}/actions/runs/9)") self.assertEqual( rm.image_link("on-prem", "v0.192.0-rc.1"), "[`on-prem:v0.192.0-rc.1`](https://console.cloud.google.com/artifacts/docker/p/l/r)", @@ -33,7 +32,6 @@ def test_plain_outside_actions(self): self.assertEqual(rm.pr_link(246), "#246") self.assertEqual(rm.commit_link(SHA), "`6635951`") self.assertEqual(rm.tag_link("v0.192.0-rc.1"), "`v0.192.0-rc.1`") - self.assertEqual(rm.run_link(9), "run 9") self.assertEqual( rm.image_link("cloud", "v0.192.0-rc.1"), "`cloud:v0.192.0-rc.1`" ) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 13046a88fa..8b4314b589 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -1,6 +1,5 @@ -"""OSS line work by git on a clone of a bare origin: cut (refuses an existing line or used minor, recovers its own -re-run), rc (tip tag else a new ee- RC; plain v* ignored), promote (open conflict PRs and the gate block unless forced, re-runnable), retire -(archive tag; RC tags go without a GA), and a tag already on another commit fails. Run: python test_oss.py +"""Tests for oss line work (cut, rc, promote, retire) on clones of a bare origin, and the workflows' +inline scripts. Run: python test_oss.py """ import contextlib @@ -15,7 +14,7 @@ from unittest import mock os.environ["GITHUB_REPOSITORY"] = ( - "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets + "Zipstack/unstract-cloud" # pin relstate to the cloud config ) import oss import relmgr as rm @@ -24,6 +23,10 @@ os.environ.pop("GITHUB_STEP_SUMMARY", None) +def dev_git(repo, *a): + return rm.git(repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a) + + class Choose(unittest.TestCase): def test_stable_else_newest_rc(self): self.assertEqual(oss.choose(["ee-v1.7.0-rc.2", "ee-v1.7.0"]), "ee-v1.7.0") @@ -42,19 +45,20 @@ def setUp(self): ) rm.git(tmp.name, "init", "-q", "--bare", "-b", "main", self.origin) rm.git(tmp.name, "clone", "-q", self.origin, self.dev) - self.g(self.dev, "commit", "-q", "--allow-empty", "-m", "base") - self.g( + dev_git(self.dev, "commit", "-q", "--allow-empty", "-m", "base") + dev_git( self.dev, "tag", "v1.7.3" ) # a community release on main: never an OSS line's tag - self.g(self.dev, "push", "-q", "origin", "main", "v1.7.3") + dev_git(self.dev, "push", "-q", "origin", "main", "v1.7.3") self.clone() self.gates = [] self.gate = 0 p = mock.patch.object( oss, "gate", - lambda o, b, ref, annotate: self.gates.append((b, ref, annotate)) - or self.gate, + lambda o, b, ref, annotate: ( + self.gates.append((b, ref, annotate)) or self.gate + ), ) p.start() self.addCleanup(p.stop) @@ -65,17 +69,14 @@ def setUp(self): p = mock.patch.object( oss.bp, "conflict_prs", - lambda *a: self.listed.append(a) - or [{"number": n, "baseRefName": b} for n, b in self.prs if b == a[1]], + lambda *a: ( + self.listed.append(a) + or [{"number": n, "baseRefName": b} for n, b in self.prs if b == a[1]] + ), ) p.start() self.addCleanup(p.stop) - def g(self, repo, *args): - return rm.git( - repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *args - ) - def clone(self): # a fresh workflow run's OSS clone rm.git( os.path.dirname(self.oss), @@ -84,23 +85,23 @@ def clone(self): # a fresh workflow run's OSS clone "--no-checkout", self.origin, self.oss, - ) if not os.path.isdir(self.oss) else self.g( + ) if not os.path.isdir(self.oss) else dev_git( self.oss, "fetch", "-q", "--prune", "--prune-tags", "--tags", "origin" ) def remote(self, *args): - return self.g(self.origin, *args) + return dev_git(self.origin, *args) def quiet(self, f, *args, **kw): with contextlib.redirect_stderr(io.StringIO()): return f(*args, **kw) def work(self, branch, msg): # a developer's commit on the OSS line, pushed - self.g(self.dev, "fetch", "-q", "origin") - self.g(self.dev, "checkout", "-q", "-B", branch, f"origin/{branch}") - self.g(self.dev, "commit", "-q", "--allow-empty", "-m", msg) - self.g(self.dev, "push", "-q", "origin", branch) - self.g(self.dev, "checkout", "-q", "main") + dev_git(self.dev, "fetch", "-q", "origin") + dev_git(self.dev, "checkout", "-q", "-B", branch, f"origin/{branch}") + dev_git(self.dev, "commit", "-q", "--allow-empty", "-m", msg) + dev_git(self.dev, "push", "-q", "origin", branch) + dev_git(self.dev, "checkout", "-q", "main") self.clone() def test_cut(self): @@ -129,7 +130,7 @@ def test_cut(self): self.work("release/1.7", "oss work") with self.assertRaisesRegex(RuntimeError, "OSS release/1.7 already exists"): oss.cut(self.oss, "1.7") - self.g(self.dev, "push", "-q", "origin", "main:refs/heads/release/1.8") + dev_git(self.dev, "push", "-q", "origin", "main:refs/heads/release/1.8") self.clone() with self.assertRaisesRegex(RuntimeError, "OSS release/1.8 already exists"): oss.cut(self.oss, "1.8") @@ -181,9 +182,9 @@ def pin(self, tag, rc): rm.git(os.path.dirname(self.cloud), "init", "-q", "-b", "main", self.cloud) with open(os.path.join(self.cloud, "pyproject.toml"), "w") as f: f.write(f'unstract-platform-version = "{tag}"\n') - self.g(self.cloud, "add", "pyproject.toml") - self.g(self.cloud, "commit", "-q", "-m", f"pin {tag}") - self.g(self.cloud, "tag", rc) + dev_git(self.cloud, "add", "pyproject.toml") + dev_git(self.cloud, "commit", "-q", "-m", f"pin {tag}") + dev_git(self.cloud, "tag", rc) def test_promote(self): self.quiet(oss.cut, self.oss, "1.7") @@ -403,19 +404,20 @@ def test_retire(self): def test_retire_keeps_a_branch_pushed_after_its_tip_was_read(self): self.quiet(oss.cut, self.oss, "1.7") self.clone() - self.g(self.dev, "fetch", "-q", "origin") # a commit lands after this run fetched - self.g(self.dev, "checkout", "-q", "-B", "release/1.7", "origin/release/1.7") - self.g(self.dev, "commit", "-q", "--allow-empty", "-m", "late fix") - self.g(self.dev, "push", "-q", "origin", "release/1.7") - late = self.g(self.dev, "rev-parse", "HEAD") - self.g(self.dev, "checkout", "-q", "main") + # a commit lands after this run fetched + dev_git(self.dev, "fetch", "-q", "origin") + dev_git(self.dev, "checkout", "-q", "-B", "release/1.7", "origin/release/1.7") + dev_git(self.dev, "commit", "-q", "--allow-empty", "-m", "late fix") + dev_git(self.dev, "push", "-q", "origin", "release/1.7") + late = dev_git(self.dev, "rev-parse", "HEAD") + dev_git(self.dev, "checkout", "-q", "main") with self.assertRaises(RuntimeError): oss.retire(self.oss, "release/1.7", "x", dry_run=False) self.assertEqual(self.remote("rev-parse", "release/1.7"), late) self.assertEqual(self.remote("tag", "-l", "ee-v1.7.*"), "ee-v1.7.0-rc.1") self.assertEqual(self.remote("tag", "-l", "retired/*"), "") self.assertEqual( - self.g(self.oss, "tag", "-l", "retired/*"), "" + dev_git(self.oss, "tag", "-l", "retired/*"), "" ) # nor left in the clone def test_retire_rerun_in_the_same_clone_archives_the_tip(self): @@ -541,9 +543,7 @@ def test_runs_on_the_oss_clone_as_the_oss_repo(self): # OSS 1.6 shipped with OSS fix F picked; OSS 1.7 lacks it. The gate reads only ee- tags and asks the OSS repo # (not the cloud one) for Backport conflict PRs. with tempfile.TemporaryDirectory() as d: - g = lambda *a: rm.git( - d, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a - ) + g = lambda *a: dev_git(d, *a) g("init", "-q", "-b", "main") g("commit", "-q", "--allow-empty", "-m", "base") g("branch", "release/1.6") @@ -599,10 +599,75 @@ def test_usage_error_is_an_error_not_a_block(self): script, ) + def test_gate_refuses_an_option_or_a_clone_outside(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "convgate.py"), "w") as fh: + fh.write("pass") + for args in ( + ("/nonexistent/oss", "release/1.7", "ee-v1.7.0-rc.1"), + (d, "-x", "ee-v1.7.0-rc.1"), + (d, "release/1.7", "-x"), + ): + with ( + self.subTest(args=args), + mock.patch.object(oss, "HERE", d), + self.assertRaises(ValueError), + ): + oss.gate(*args, "error") + + def test_main_refuses_an_option_as_a_ref(self): + with tempfile.TemporaryDirectory() as d: + for cmd in ( + ["cut", "--line=-x"], + ["rc", "--branch=-x"], + ["promote", "--rc=-x"], + ): + with ( + self.subTest(cmd=cmd), + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit), + ): + oss.main(["--oss-repo", d, *cmd]) + + def test_main_reports_a_clone_outside(self): + with contextlib.redirect_stderr(io.StringIO()) as err: + code = oss.main(["--oss-repo", "/nonexistent/oss", "cut", "--line", "1.7"]) + self.assertEqual(code, 1) + self.assertIn("::error::", err.getvalue()) + HERE = os.path.dirname(os.path.abspath(__file__)) +class TopicBranchHead(unittest.TestCase): + """release-branch-only.yaml's head check: the bot never pushes to main, a release line or a backport head.""" + + def test_head_ref(self): + with open( + os.path.join(HERE, "..", "workflows", "release-branch-only.yaml") + ) as fh: + wf = fh.read() + body = wf[wf.index("run: |", wf.index("- name: Topic-branch head\n")) :].split( + "\n" + )[1:] + script = "\n".join(line[10:] for line in body[: body.index("")]) + for head, code in ( + ("main", 1), + ("release/1.7", 1), + ("backport/1.7/abc1234", 1), + ("feature/x", 0), + ("x-main", 0), + ): + with self.subTest(head=head): + r = subprocess.run( + ["bash", "-c", script], + env={"HEAD_REF": head, "PATH": os.environ["PATH"]}, + capture_output=True, + text=True, + ) + self.assertEqual(r.returncode, code, r.stdout) + + def inline(workflow, step): """The inline python of a workflow step.""" with open(os.path.join(HERE, "..", "workflows", workflow)) as fh: @@ -620,10 +685,9 @@ def inline(workflow, step): "the cloud repo's workflows", ) class Workflow(unittest.TestCase): - """A cloud and an OSS origin with clones; a sitecustomize stands in for Artifact Registry with reg.jsonl as - cloud/backend's versions and reg-_.jsonl as another artifact's (empty when absent); a stub gh - serves the open backport-conflict PRs of each repo (_.json; a `prfail` file fails the listing) and - the PRs of a main commit (pulls.json). gh.log: each pr list and commit PRs call, as its token. + """Cloud and OSS origins with clones. sitecustomize serves Artifact Registry from reg.jsonl (cloud/backend) + and reg-_.jsonl; a stub gh serves _.json (conflict PRs; a `prfail` file fails it) + and pulls.json, logging each call and its token to gh.log. """ def setUp(self): @@ -635,8 +699,8 @@ def setUp(self): origin, work = os.path.join(d, f"{name}.git"), os.path.join(d, name) rm.git(d, "init", "-q", "--bare", "-b", "main", origin) rm.git(d, "clone", "-q", origin, work) - self.g(work, "commit", "-q", "--allow-empty", "-m", "base") - self.g(work, "push", "-q", "origin", "main") + dev_git(work, "commit", "-q", "--allow-empty", "-m", "base") + dev_git(work, "push", "-q", "origin", "main") self.repos[name] = (origin, work) os.makedirs(os.path.join(self.cloud, ".relmgr", ".github")) os.symlink(HERE, os.path.join(self.cloud, ".relmgr", ".github", "release")) @@ -673,16 +737,13 @@ def _ar_versions(artifact): # reg.jsonl for cloud/backend, reg-_ cloud = property(lambda self: self.repos["cloud"][1]) - def g(self, repo, *a): - return rm.git(repo, "-c", "user.name=Dev", "-c", "user.email=dev@example.com", *a) - def line( self, x_y, *tags ): # cloud and OSS release/X.Y; tags: cloud tags on the cloud tip, also in the registry - self.g(self.cloud, "push", "-q", "origin", f"main:refs/heads/release/{x_y}") + dev_git(self.cloud, "push", "-q", "origin", f"main:refs/heads/release/{x_y}") for i, t in enumerate(tags): - self.g(self.cloud, "tag", t, "main") - self.g(self.cloud, "push", "-q", "origin", t) + dev_git(self.cloud, "tag", t, "main") + dev_git(self.cloud, "push", "-q", "origin", t) with open(os.path.join(self.d, "reg.jsonl"), "a") as fh: fh.write( f'{{"id": {len(t) * 10 + i}, "digest": "sha256:{t}", "pushed_at": "2026-09-01T00:00:00Z", ' @@ -692,7 +753,7 @@ def line( oss.cut(self.repos["oss"][1], x_y) def run_step(self, script, **env): - self.g(self.cloud, "fetch", "-q", "--tags", "origin") + dev_git(self.cloud, "fetch", "-q", "--tags", "origin") for name in ("output", "notes.md"): # a fresh run's files with contextlib.suppress(FileNotFoundError): os.remove(os.path.join(self.d, name)) @@ -788,8 +849,8 @@ def moved( origin = self.repos["cloud"][0] tip = rm.git(origin, "rev-parse", f"release/{x_y}") rm.git(origin, "tag", f"retired/{x_y}-{tip[:7]}", tip) - self.g(self.cloud, "commit", "-q", "--allow-empty", "-m", "late fix") - self.g(self.cloud, "push", "-q", "origin", f"HEAD:refs/heads/release/{x_y}") + dev_git(self.cloud, "commit", "-q", "--allow-empty", "-m", "late fix") + dev_git(self.cloud, "push", "-q", "origin", f"HEAD:refs/heads/release/{x_y}") return f"retired/{x_y}-{tip[:7]}" def refused(self, x_y, tag, **env): @@ -808,10 +869,10 @@ def lands_mid_run(self, x_y): run's first other push (an RC tag delete or the archive tag), before the branch delete. """ origin = self.repos["cloud"][0] - self.g(self.cloud, "commit", "-q", "--allow-empty", "-m", "late fix") - late = self.g(self.cloud, "rev-parse", "HEAD") - self.g(self.cloud, "push", "-q", "origin", "HEAD:refs/keep/late") - self.g(self.cloud, "reset", "-q", "--hard", "HEAD~1") + dev_git(self.cloud, "commit", "-q", "--allow-empty", "-m", "late fix") + late = dev_git(self.cloud, "rev-parse", "HEAD") + dev_git(self.cloud, "push", "-q", "origin", "HEAD:refs/keep/late") + dev_git(self.cloud, "reset", "-q", "--hard", "HEAD~1") hook = os.path.join(origin, "hooks", "post-receive") with open(hook, "w") as fh: fh.write( @@ -868,18 +929,18 @@ def test_derived_list_refuses_a_line_moved_after_its_archive(self): class PendingPicksWorkflow(Workflow): - """Promote's Plan blocks on open backport-conflict PRs of the cloud line or the OSS line it pins, and on held - security/critical fixes, unless forced (recorded). A failed listing blocks too: forced, a failed conflict listing - passes and is recorded, a failed held listing still blocks. Take RC only warns. + """Promote's Plan blocks on open conflict PRs (cloud line or pinned OSS line) and held security/critical fixes + unless forced. A failed listing blocks; forcing passes a failed conflict listing (recorded), never a failed + held listing. Take RC only warns. """ def setUp(self): super().setUp() with open(os.path.join(self.cloud, "pyproject.toml"), "w") as fh: fh.write('unstract-platform-version = "ee-v1.7.0-rc.1"\n') - self.g(self.cloud, "add", "pyproject.toml") - self.g(self.cloud, "commit", "-q", "-m", "pin") - self.g(self.cloud, "push", "-q", "origin", "main") + dev_git(self.cloud, "add", "pyproject.toml") + dev_git(self.cloud, "commit", "-q", "-m", "pin") + dev_git(self.cloud, "push", "-q", "origin", "main") def prs(self, slug, *bases): self.write( @@ -959,11 +1020,11 @@ def test_promote_fails_closed(self): def test_promote_held_fixes(self): self.line("1.7", "v1.7.0-rc.1", "v1.7.0") # cloud GA: the line is stabilized - self.g( + dev_git( self.cloud, "commit", "-q", "--allow-empty", "-m", "fix: y" ) # the next RC's commit - self.g(self.cloud, "tag", "v1.7.1-rc.1") - self.g( + dev_git(self.cloud, "tag", "v1.7.1-rc.1") + dev_git( self.cloud, "push", "-q", @@ -975,10 +1036,10 @@ def test_promote_held_fixes(self): fh.write( '{"id": 9, "digest": "sha256:rc", "pushed_at": "2026-09-01T00:00:00Z", "tags": ["v1.7.1-rc.1"]}\n' ) - self.g( + dev_git( self.cloud, "commit", "-q", "--allow-empty", "-m", "fix: x (#9)" ) # on main only: held for 1.7 - self.g(self.cloud, "push", "-q", "origin", "main") + dev_git(self.cloud, "push", "-q", "origin", "main") self.write( "pulls.json", json.dumps( @@ -1000,10 +1061,10 @@ def test_promote_held_fixes(self): oss_work = self.repos["oss"][ 1 ] # a security fix on OSS main, not picked onto OSS release/1.7: held too - self.g(oss_work, "pull", "-q", "origin", "main") - self.g(oss_work, "commit", "-q", "--allow-empty", "-m", "fix: z (#9)") - self.g(oss_work, "push", "-q", "origin", "main") - self.g(oss_work, "fetch", "-q", "origin") + dev_git(oss_work, "pull", "-q", "origin", "main") + dev_git(oss_work, "commit", "-q", "--allow-empty", "-m", "fix: z (#9)") + dev_git(oss_work, "push", "-q", "origin", "main") + dev_git(oss_work, "fetch", "-q", "origin") r = self.promote("v1.7.1-rc.1") self.blocked(r, "::error::pending picks") self.assertIn(f"oss-token api repos/{oss.OSS}/commits/", self.read("gh.log")) @@ -1055,7 +1116,7 @@ def test_take_rc_only_warns(self): self.line("1.7") self.prs(oss.OSS, "release/1.7") script = inline("take-rc-build.yaml", "Plan checks") - self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + dev_git(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") r = self.run_step(script, BRANCH="release/1.7", TARGETS="cloud") self.assertEqual(r.returncode, 0, r.stdout + r.stderr) self.assertIn( @@ -1074,7 +1135,7 @@ class TakeRcVersion(Workflow): def test_allocates_past_a_registry_rc(self): # An earlier run pushed the cloud artifact of v1.7.0-rc.1, then its tag push failed and the tip moved. self.line("1.7") - self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + dev_git(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") self.write( "reg.jsonl", '{"id": 1, "digest": "sha256:a", "pushed_at": "2026-09-01T00:00:00Z", "tags": ["v1.7.0-rc.1"]}\n', @@ -1093,7 +1154,7 @@ def test_allocates_past_a_registry_rc(self): def test_patch_rc_on_a_ga_line_does_not_reopen(self): # Every enabled target (cloud) is GA: a cloud patch RC keeps the OSS line's stabilized tag. self.line("1.7", "v1.7.0") - self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + dev_git(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") r = self.run_step( inline("take-rc-build.yaml", "Plan version and targets"), BRANCH="release/1.7", @@ -1106,15 +1167,15 @@ def test_patch_rc_on_a_ga_line_does_not_reopen(self): ) def test_rerun_on_a_tagged_tip_reopens_for_a_target_already_built(self): - # An earlier run tagged v1.7.1-rc.1 for cloud and on-prem on a line GA for cloud only; its on-prem build - # pushed, its cloud build failed. A fresh run builds only cloud, yet on-prem still has no GA: reopen. + # Tip tagged v1.7.1-rc.1 for both targets, on-prem built, cloud failed; a cloud-only rerun still + # reopens (on-prem has no GA). self.line("1.7", "v1.7.0") - self.g(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") - self.g( + dev_git(self.cloud, "checkout", "-q", "-b", "release/1.7", "origin/release/1.7") + dev_git( self.cloud, "commit", "-q", "--allow-empty", "-m", "release: chart 1.7.1-rc.1" ) - self.g(self.cloud, "tag", "v1.7.1-rc.1") - self.g( + dev_git(self.cloud, "tag", "v1.7.1-rc.1") + dev_git( self.cloud, "push", "-q", @@ -1150,12 +1211,12 @@ def release( if pin: with open(path, "w") as fh: fh.write(f'unstract-platform-version = "{pin}"\n') - self.g(self.cloud, "add", "pyproject.toml") + dev_git(self.cloud, "add", "pyproject.toml") elif os.path.exists(path): - self.g(self.cloud, "rm", "-q", "pyproject.toml") - self.g(self.cloud, "commit", "-q", "--allow-empty", "-m", f"pin {pin}") + dev_git(self.cloud, "rm", "-q", "pyproject.toml") + dev_git(self.cloud, "commit", "-q", "--allow-empty", "-m", f"pin {pin}") for t in tags: - self.g( + dev_git( self.cloud, "tag", *( @@ -1168,7 +1229,7 @@ def release( fh.write( f'{{"id": {len(t)}, "digest": "sha256:{t}", "pushed_at": "2026-09-01T00:00:00Z", "tags": ["{t}"]}}\n' ) - self.g(self.cloud, "push", "-q", "--tags", "origin", "main") + dev_git(self.cloud, "push", "-q", "--tags", "origin", "main") def plan(self): r = self.run_step( diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index a486220ef5..d8acd4b50b 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -1,6 +1,4 @@ -"""Synthetic-repo tests for relstate (derive 1-11, on-prem newest six 7b, cutover fallback 12, retired-line history 13, tip_rc 14, summary 15, tip_rc Slack link 20, Release-branch-only line in a squash message 21, abandon_plan 16-17, pr_fix_class 18, revert titles 18b, ticketless titles 18c, Release-branch-only key 18d, release-branch-only label 18e, ahead_of_ga 19, promote order per target 22, enable a target on the newest line only 23, no order block on a GA line 23b, already_bumped 24, promoted from the build artifact 25, OSS repo config 26 (ee- git tags, plain v* ignored, lines kept while the branch exists), next_minor past a stable tag on main 27, bump with no chart 28, retire_line refusals and the retired line's minor 29, RC allocation past registry RCs 30, a reused minor past its kept RC 31, shallow clone refused 32, conflict-marker grep error 33, a revert of another line's pick 34, release-branch CI commit selection 35, backport head names 36, unknown repo config 37, Artifact Registry reads 38-41 and 39b, a chart push without v 42). Run: python test_relstate.py. release-tooling-tests.yaml runs all six on a pull request -that changes this directory. -""" +"""Synthetic-repo tests for relstate and relmgr. Run: python test_relstate.py; release-tooling-tests.yaml runs every test_*.py here on a PR that changes this directory.""" import contextlib import datetime as dt @@ -14,12 +12,12 @@ import urllib.parse os.environ["GITHUB_REPOSITORY"] = ( - "Zipstack/unstract-cloud" # relstate loads the cloud config, whatever the runner sets + "Zipstack/unstract-cloud" # pin relstate to the cloud config ) import relmgr as rm import relstate as rs -# Isolate from the user's git config (signing, noprefix, hooks...). +# ignore the user's git config os.environ.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") os.environ.pop("GITHUB_OUTPUT", None) NOW = dt.datetime(2026, 9, 29, tzinfo=dt.UTC) @@ -73,25 +71,25 @@ def branch(self, st, name): def ga(self, artifact, tag): return {"tag": tag, **self.reg[artifact][tag]} - # Scenarios build on each other as the table does. - def s1(self): + # Each scenario builds on the previous one. + def cut_with_rc(self): self.cut("1.5") self.push("cloud/backend", "v1.5.0-rc.1", git_ref="release/1.5") - def s2(self): - self.s1() + def cloud_ga(self): + self.cut_with_rc() self.push("cloud/backend", "v1.5.0", git_ref="release/1.5") - def s3(self): - self.s2() + def onprem_rc(self): + self.cloud_ga() self.push("on-prem/backend", "v1.5.0-rc.3", git_ref="release/1.5") - def s4(self, chart_days=10): - self.s3() + def onprem_ga(self, chart_days=10): + self.onprem_rc() self.push("chart/unstract-platform", "v1.5.1", days=chart_days) - def s6(self, chart_days=10): - self.s4(chart_days) + def newer_cloud_ga(self, chart_days=10): + self.onprem_ga(chart_days) self.cut("1.6") self.push("cloud/backend", "v1.6.0-rc.1", git_ref="release/1.6") self.push("cloud/backend", "v1.6.0", git_ref="release/1.6") @@ -108,8 +106,8 @@ def onprem(self, state, chart=True): "promoted": None, } - def test_1_cut_with_cloud_rc(self): - self.s1() + def test_cut_with_cloud_rc(self): + self.cut_with_rc() st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -133,8 +131,8 @@ def test_1_cut_with_cloud_rc(self): self.assertEqual(st["backport_targets"]["fix"], ["release/1.5"]) self.assertEqual(st["supported"], {"cloud": ["release/1.5"], "on-prem": []}) - def test_2_cloud_ga(self): - self.s2() + def test_cloud_ga(self): + self.cloud_ga() st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -158,8 +156,8 @@ def test_2_cloud_ga(self): }, ) - def test_3_onprem_rc_reopens_branch(self): - self.s3() + def test_onprem_rc_reopens_branch(self): + self.onprem_rc() st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -178,8 +176,8 @@ def test_3_onprem_rc_reopens_branch(self): ) self.assertEqual(st["backport_targets"]["fix"], ["release/1.5"]) - def test_4_both_stabilized(self): - self.s4() + def test_both_stabilized(self): + self.onprem_ga() st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -198,8 +196,8 @@ def test_4_both_stabilized(self): ) self.assertEqual(st["supported"]["on-prem"], ["release/1.5"]) - def test_5_onprem_window_over(self): - self.s4(chart_days=200) + def test_onprem_window_over(self): + self.onprem_ga(chart_days=200) st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -219,16 +217,16 @@ def test_5_onprem_window_over(self): self.assertEqual(st["supported"], {"cloud": ["release/1.5"], "on-prem": []}) self.assertEqual(st["retire"], []) - def test_5b_onprem_window_from_first_ga(self): + def test_onprem_window_from_first_ga(self): # A patch chart push does not extend the window: it runs from the line's first stable chart push. - self.s4(chart_days=200) + self.onprem_ga(chart_days=200) self.push("chart/unstract-platform", "v1.5.2", days=1) tt = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] self.assertEqual(tt["state"], "past-eos") self.assertEqual(tt["ga"], self.ga("chart/unstract-platform", "v1.5.1")) - def test_6_newer_cloud_ga(self): - self.s6() + def test_newer_cloud_ga(self): + self.newer_cloud_ga() st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -249,8 +247,8 @@ def test_6_newer_cloud_ga(self): st["supported"], {"cloud": ["release/1.6"], "on-prem": ["release/1.5"]} ) - def test_7_retire(self): - self.s6(chart_days=200) + def test_retire(self): + self.newer_cloud_ga(chart_days=200) st = self.state() self.assertEqual( self.branch(st, "release/1.5"), @@ -270,7 +268,7 @@ def test_7_retire(self): self.assertEqual(st["retire"], ["release/1.5"]) self.assertEqual(st["supported"], {"cloud": ["release/1.6"], "on-prem": []}) - def test_7b_onprem_newest_six(self): + def test_onprem_newest_six(self): for y in range(1, 9): # 1.1 .. 1.8, each with an on-prem chart inside the window self.cut(f"1.{y}") self.push("on-prem/backend", f"v1.{y}.0-rc.1", git_ref=f"release/1.{y}") @@ -283,8 +281,8 @@ def test_7b_onprem_newest_six(self): ) self.assertEqual(st["retire"], ["release/1.2", "release/1.1"]) - def test_8_cut_no_artifacts(self): - self.s4() + def test_cut_no_artifacts(self): + self.onprem_ga() self.cut("1.7") st = self.state() self.assertEqual( @@ -303,8 +301,8 @@ def test_8_cut_no_artifacts(self): self.assertTrue(all("release/1.7" not in v for v in st["supported"].values())) self.assertIn("release/1.7", st["backport_targets"]["fix"]) - def test_9_tag_not_on_branch(self): - self.s1() + def test_tag_not_on_branch(self): + self.cut_with_rc() self.push("on-prem/backend", "v1.5.0-rc.9", git_ref=self.commit("main-only")) st = self.state() self.assertEqual(self.branch(st, "release/1.5")["enabled"], ["cloud"]) @@ -321,8 +319,8 @@ def test_9_tag_not_on_branch(self): ], ) - def test_10_registry_failure(self): - self.s1() + def test_registry_failure(self): + self.cut_with_rc() def boom(artifact): raise RuntimeError("HTTP 403") @@ -336,7 +334,7 @@ def boom(artifact): self.assertEqual(out.getvalue(), "") self.assertIn("HTTP 403", err.getvalue()) - def test_11_next_minor_ignores_legacy(self): + def test_next_minor_ignores_legacy(self): self.cut("1.5") self.cut("1.6") self.g("branch", "release/v0.148.1001", "main") @@ -346,8 +344,8 @@ def test_11_next_minor_ignores_legacy(self): [b["branch"] for b in st["branches"]], ["release/1.6", "release/1.5"] ) - def test_13_retired_lines_are_history_not_anomalies(self): - self.s4() # only release/1.5 is live + def test_retired_lines_are_history_not_anomalies(self): + self.onprem_ga() # only release/1.5 is live retired = self.commit("1.6 work") self.push( "cloud/backend", "v1.6.0-rc.1", git_ref=retired @@ -373,8 +371,8 @@ def test_13_retired_lines_are_history_not_anomalies(self): ) self.assertEqual([b["branch"] for b in st["branches"]], ["release/1.5"]) - def test_14_next_rc_moves_after_tip_is_built(self): - self.s2() + def test_next_rc_moves_after_tip_is_built(self): + self.cloud_ga() self.commit("pick", "release/1.5") self.assertEqual( { @@ -384,8 +382,8 @@ def test_14_next_rc_moves_after_tip_is_built(self): {"next_rc": "v1.5.1-rc.1", "tip_rc": None}, ) - def test_15_summary(self): - self.s6() + def test_summary(self): + self.newer_cloud_ga() out = rs.summary(self.state()) self.assertIn( "| release/1.6 | stabilized | stabilized (GA v1.6.0) | - | v1.6.0-rc.1 | - | 0 |", @@ -397,8 +395,8 @@ def test_15_summary(self): ) self.assertIn("next_minor: 1.7", out) - def test_19_ahead_of_ga(self): - self.s2() + def test_ahead_of_ga(self): + self.cloud_ga() self.g("tag", "v1.5.1", "release/1.5") # latest stable tag counts, not the first self.commit("pick 1", "release/1.5") self.commit("pick 2", "release/1.5") @@ -410,27 +408,27 @@ def test_19_ahead_of_ga(self): ) self.assertIn("| v1.5.2-rc.1 | 2 |", rs.summary(st)) - def test_21_rbo_line_in_squash_message(self): + def test_rbo_line_in_squash_message(self): self.cut("1.5") squash = ( - "UN-37 [MISC] pin redis (#9)\n\n* chore: pin redis\n\nwhy it is pinned\n\n" - "* Release-branch-only: UN-37\n\n---------\n\nCo-authored-by: x " + "UN-1 [MISC] pin redis (#9)\n\n* chore: pin redis\n\nwhy it is pinned\n\n" + "* Release-branch-only: UN-1\n\n---------\n\nCo-authored-by: x " ) cases = { squash: "release-branch-only", # COMMIT_MESSAGES squash: the bot's subject-only commit as `* ...` - "fix: pin\n\nbody\n\nRelease-branch-only: UN-37": "release-branch-only", # a trailer - "fix: pin\n\nsee the Release-branch-only: UN-37 note": "invalid", # mid-line is not the line - "fix: pin\n\n** Release-branch-only: UN-37": "invalid", + "fix: pin\n\nbody\n\nRelease-branch-only: UN-1": "release-branch-only", # a trailer + "fix: pin\n\nsee the Release-branch-only: UN-1 note": "invalid", # mid-line is not the line + "fix: pin\n\n** Release-branch-only: UN-1": "invalid", } for msg, kind in cases.items(): sha = self.commit(msg, "release/1.5") self.assertEqual(rm.classify_commit(self.repo, sha), kind, msg) - self.assertEqual(rm.RBO_RE.findall(squash), ["UN-37"]) + self.assertEqual(rm.RBO_RE.findall(squash), ["UN-1"]) - def test_22_promote_blockers(self): + def test_promote_blockers(self): """Promote order per target: only an older line with the target enabled and not GA for it blocks.""" msg = "release/{} is not GA for {} yet: promote and publish it first, retire it, or force with a reason" - self.s3() # 1.5: cloud GA, on-prem RC only + self.onprem_rc() # 1.5: cloud GA, on-prem RC only self.cut("1.6") self.push("cloud/backend", "v1.6.0-rc.1", git_ref="release/1.6") self.cut("1.7") # nothing built yet @@ -464,12 +462,12 @@ def test_22_promote_blockers(self): self.g("branch", "-D", "release/1.6") self.assertEqual(rs.promote_blockers(self.state(), "release/1.7", "cloud"), []) - def test_23_enable_blockers(self): + def test_enable_blockers(self): """A target is enabled only on a line newer than every line already GA for it.""" msg = "release/{} is older than release/{}, already GA for {}: enable {} on a newer line instead" self.cut("1.4") self.push("cloud/backend", "v1.4.0-rc.1", git_ref="release/1.4") - self.s6() # 1.5: cloud GA, on-prem GA (chart v1.5.1); 1.6: cloud GA, no on-prem + self.newer_cloud_ga() # 1.5: cloud GA, on-prem GA (chart v1.5.1); 1.6: cloud GA, no on-prem st = self.state() # (a) an older line enabling on-prem behind a line GA for on-prem is refused self.assertEqual( @@ -482,8 +480,8 @@ def test_23_enable_blockers(self): self.assertEqual(rs.enable_blockers(st, "release/1.4", "cloud"), []) self.assertEqual(rs.enable_blockers(st, "release/1.5", "on-prem"), []) - def test_23b_no_order_block_on_a_ga_line(self): - """(d) Invariant: in a state reached only by moves the enable rule and the promote order allow, a patch on a + def test_no_order_block_on_a_ga_line(self): + """Invariant: in a state reached only by moves the enable rule and the promote order allow, a patch on a line already GA for a target is never order-blocked. Without the enable rule it can be. """ rc, ga = ( @@ -573,7 +571,7 @@ def chart(self, ver, branch, msg, email="dev@example.com"): ) self.g("checkout", "-q", "main") - def test_24_already_bumped(self): + def test_already_bumped(self): self.cut("1.5") self.chart("1.5.0-rc.1", "release/1.5", "release: chart 1.5.0-rc.1", rm.BOT_EMAIL) self.assertTrue(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.1")) @@ -589,8 +587,8 @@ def test_24_already_bumped(self): ) # right subject, not the bot self.assertFalse(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.2")) - def test_25_promoted_from_build_artifact(self): - self.s3() + def test_promoted_from_build_artifact(self): + self.onprem_rc() self.push( "on-prem/backend", "v1.5.1", git_ref="release/1.5" ) # Promote on-prem, chart not published @@ -622,8 +620,8 @@ def test_25_promoted_from_build_artifact(self): onprem = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] self.assertEqual(onprem["promoted"], self.ga("on-prem/backend", "v1.5.2")) - def test_20_tip_rc_slack(self): - self.s1() + def test_tip_rc_slack(self): + self.cut_with_rc() self.assertNotIn( "tip_rc_slack", self.branch(self.state(), "release/1.5") ) # lightweight tag: no key @@ -643,7 +641,7 @@ def test_20_tip_rc_slack(self): self.assertEqual(self.branch(st, "release/1.5")["tip_rc_slack"], url) self.assertIn(f"| v1.5.0-rc.1 ([Slack]({url})) |", rs.summary(st)) - def test_12_next_minor_cutover_fallback(self): + def test_next_minor_cutover_fallback(self): self.g("tag", "v0.181.0") self.g("tag", "v0.181.1001") self.g("tag", "v0.182.0-rc.1") # RC tags do not count @@ -680,7 +678,7 @@ def built_by_mistake(self): "cloud/backend", "v1.5.0" ) # other versions keep the cloud package non-empty - def test_16_abandon_plan_built_by_mistake(self): + def test_abandon_plan_built_by_mistake(self): self.built_by_mistake() self.push("cloud/backend", "v1.9.0-rc.2") self.reg["cloud/backend"]["edge"] = self.reg["cloud/backend"][ @@ -707,7 +705,16 @@ def test_16_abandon_plan_built_by_mistake(self): rs.abandon_plan(self.repo, "release/1.8", lambda a: {})["versions"], {} ) - def test_17_abandon_plan_refusals(self): + def test_abandon_plan_keeps_the_newest_rc(self): + self.built_by_mistake() + for n in range(2, 11): + self.push("on-prem/backend", f"v1.9.0-rc.{n}") + plan = rs.abandon_plan(self.repo, "release/1.9", lambda a: self.reg.get(a, {})) + self.assertIn("on-prem/backend:v1.9.0-rc.10", plan["kept"]) + dropped = [t for v in plan["versions"]["on-prem/backend"] for t in v["tags"]] + self.assertEqual(dropped, [f"v1.9.0-rc.{n}" for n in range(1, 10)]) + + def test_abandon_plan_refusals(self): self.built_by_mistake() plan = lambda: rs.abandon_plan( self.repo, "release/1.9", lambda a: self.reg.get(a, {}) @@ -719,9 +726,9 @@ def test_17_abandon_plan_refusals(self): human = self.commit("human RBO-less work", "release/1.9") self.assertEqual(plan()["used"], [human[:12]]) - def test_42_chart_push_without_v(self): + def test_chart_push_without_v(self): # Publish pushes the chart as its Chart.yaml version, X.Y.Z: it anchors on-prem GA as vX.Y.Z. - self.s3() + self.onprem_rc() self.push("chart/unstract-platform", "1.5.1") tt = self.branch(self.state(), "release/1.5")["targets"]["on-prem"] self.assertEqual( @@ -747,12 +754,12 @@ def test_42_chart_push_without_v(self): ["chart/unstract-platform:v1.9.0"], ) - def test_43_stabilizes_when_every_enabled_target_is_ga(self): + def test_stabilizes_when_every_enabled_target_is_ga(self): # Cloud Promote marks the OSS line stabilized/X.Y on the promote that leaves every enabled target GA. def row(): return self.branch(self.state(), "release/1.5") - self.s1() # a cloud-only line: its cloud GA + self.cut_with_rc() # a cloud-only line: its cloud GA self.assertTrue(rs.stabilizes(row(), "cloud")) self.push("on-prem/backend", "v1.5.0-rc.1") # on-prem enabled, nothing GA self.assertEqual(row()["enabled"], ["cloud", "on-prem"]) @@ -767,12 +774,12 @@ def row(): # A later patch: oss.promote keeps the tag. self.assertTrue(rs.stabilizes(row(), "cloud")) - def test_44_reopens_when_a_built_target_has_no_ga(self): + def test_reopens_when_a_built_target_has_no_ga(self): # Take RC deletes the OSS line's stabilized tag when it builds a target with no GA on the line. def row(): return self.branch(self.state(), "release/1.5") - self.s2() # cloud GA, cloud only + self.cloud_ga() # cloud GA, cloud only self.assertFalse(rs.reopens(row(), ["cloud"])) # a cloud patch RC self.assertTrue(rs.reopens(row(), ["on-prem"])) # on-prem enabled late self.assertTrue(rs.reopens(row(), ["cloud", "on-prem"])) @@ -780,7 +787,7 @@ def row(): self.push("chart/unstract-platform", "1.5.1") # on-prem GA too self.assertFalse(rs.reopens(row(), ["cloud", "on-prem"])) - def test_26_oss_repo_config(self): + def test_oss_repo_config(self): # The OSS repo runs this same file: one target, oss, read from the line's own ee- git tags (no registry), # supported while its branch exists once its stabilized/X.Y tag is there. Plain v* tags are community releases from main and never count. self.addCleanup(rs.use, rs.SLUG) @@ -846,7 +853,7 @@ def test_26_oss_repo_config(self): ], ) - def test_27_next_minor_past_stable_tag_on_main(self): + def test_next_minor_past_stable_tag_on_main(self): # A minor already shipped from main (tags before the first line) is never cut again. self.g("tag", "v1.5.14") self.commit("more main work") @@ -854,7 +861,7 @@ def test_27_next_minor_past_stable_tag_on_main(self): self.g("branch", "release/1.5", "v1.5.14") self.assertEqual(self.state()["next_minor"], "1.7") - def test_28_already_bumped_no_chart(self): + def test_already_bumped_no_chart(self): self.cut("1.5") def bump(msg): @@ -878,11 +885,11 @@ def bump(msg): bump("release: chart 1.5.0-rc.2") # the chart subject needs a chart self.assertFalse(rm.already_bumped(self.repo, "release/1.5", "v1.5.0-rc.2")) - def test_29_retire_line(self): + def test_retire_line(self): refusal = lambda force=False: rs.retire_line_refusal( self.state(), "release/1.5", force ) - self.s3() # cloud GA newest (production), on-prem RCs only + self.onprem_rc() # cloud GA newest (production), on-prem RCs only self.assertEqual( refusal(True), "cloud is production (newest cloud GA); it retires once a newer line is promoted", @@ -916,7 +923,7 @@ def pick_of(self, fix): # a non-empty pick of fix on the checked-out branch self.g("add", "p.txt") self.g("commit", "-q", "-m", f"fix on main\n\n(cherry picked from commit {fix})") - def test_30_rc_version_skips_registry_rcs(self): + def test_rc_version_skips_registry_rcs(self): # An artifact pushed by a run whose tag push failed: its RC is never allocated again once the tip moves. self.cut("1.5") taken = {"v1.5.0-rc.1", "v1.5.0-rc.2", "v1.4.0-rc.9", "v1.5.1-rc.7", "edge"} @@ -948,9 +955,9 @@ def test_30_rc_version_skips_registry_rcs(self): self.branch(self.state(), "release/1.5")["next_rc"], "v1.5.0-rc.5" ) - def test_31_reused_minor_skips_its_kept_rc(self): - # release/1.7 built once, then abandoned: its git RC tag deleted, its registry RC kept (the package's last - # version). The minor is cut again, and its first RC is past the kept one. + def test_reused_minor_skips_its_kept_rc(self): + # 1.7 abandoned: git RC tag deleted, registry RC kept (package's last version); the re-cut's + # first RC skips it. self.cut("1.6") tip = self.cut("1.7") self.push("on-prem/backend", "v1.7.0-rc.1") @@ -962,7 +969,7 @@ def test_31_reused_minor_skips_its_kept_rc(self): self.branch(self.state(), "release/1.7")["next_rc"], "v1.7.0-rc.2" ) - def test_32_shallow_clone_refused(self): + def test_shallow_clone_refused(self): clone = os.path.join(self._tmp.name, "shallow") self.commit("second") rm.git( @@ -972,7 +979,7 @@ def test_32_shallow_clone_refused(self): rm.git(clone, "log", "-1") self.assertNotIn(clone, rm._FULL) - def test_33_conflict_markers_git_error(self): + def test_conflict_markers_git_error(self): with open(os.path.join(self.repo, "f.txt"), "w") as f: f.write("a\n<<<<<<< ours\nb\n=======\nc\n>>>>>>> theirs\n") self.g("add", "f.txt") @@ -981,7 +988,7 @@ def test_33_conflict_markers_git_error(self): with self.assertRaisesRegex(RuntimeError, r"git grep \(exit 128\)"): rm.conflict_markers(self.repo, "no-such-rev") - def test_34_reverted_pick_only_on_its_own_branch(self): + def test_reverted_pick_only_on_its_own_branch(self): fix = self.commit("fix on main") self.cut("1.5") self.cut("1.6") @@ -1005,7 +1012,7 @@ def test_34_reverted_pick_only_on_its_own_branch(self): ) ) - def test_35b_forged_pick_trailer_is_invalid(self): + def test_forged_pick_trailer_is_invalid(self): # A trailer naming a commit that is not on main is not a pick of a main original. self.cut("1.5") work = self.g("rev-parse", "release/1.5") # on the line only @@ -1020,7 +1027,7 @@ def test_35b_forged_pick_trailer_is_invalid(self): [(forged, "invalid")], ) - def test_35_ci_commits(self): + def test_ci_commits(self): fix = self.commit("fix on main") self.cut("1.5") # one invalid commit, `work on 1.5` before = self.g("rev-parse", "release/1.5") @@ -1102,7 +1109,7 @@ def test_35_ci_commits(self): ) # only a backport PR's tree is checked self.g("checkout", "-q", "main") - def test_36_is_backport(self): + def test_is_backport(self): for ref, want in ( ("backport/1.5/abcdef0", True), ("backport/0.192/1234567", True), @@ -1115,7 +1122,7 @@ def test_36_is_backport(self): ): self.assertEqual(rm.is_backport(ref), want, ref) - def test_37_unknown_repo_config(self): + def test_unknown_repo_config(self): with self.assertRaisesRegex( RuntimeError, "relstate has no config for repo 'nope'" ): @@ -1126,7 +1133,7 @@ def test_37_unknown_repo_config(self): class PrFixClass(unittest.TestCase): """Lint PR / Backport class rule: title type plus fix/ labels.""" - def test_18_pr_fix_class(self): + def test_pr_fix_class(self): c = rm.pr_fix_class self.assertEqual(c("UN-1 [FIX] x"), "fix") self.assertIsNone(c("UN-1 [MISC] x")) @@ -1152,7 +1159,7 @@ def test_18_pr_fix_class(self): "fix/security, fix/critical, fix/needs-backport", str(e.exception) ) - def test_18b_revert_takes_inner_class(self): + def test_revert_takes_inner_class(self): c = rm.pr_fix_class self.assertEqual(c('Revert "UN-1 [FIX] x"'), "fix") self.assertIsNone(c('Revert "UN-1 [MISC] x"')) @@ -1164,7 +1171,7 @@ def test_18b_revert_takes_inner_class(self): c("no prefix") self.assertEqual(c("no prefix", ["fix/needs-backport"]), "fix") - def test_18c_ticketless_title_same_class(self): + def test_ticketless_title_same_class(self): c = rm.pr_fix_class self.assertEqual(c("[FIX] x"), "fix") self.assertEqual(c("[HOT-FIX] x"), "fix") @@ -1179,7 +1186,7 @@ def test_18c_ticketless_title_same_class(self): with self.assertRaises(ValueError, msg=title): c(title) - def test_18e_release_only_label_refused_off_release(self): + def test_release_only_label_refused_off_release(self): f = rm.release_only_refusal self.assertIn( "for PRs into a release/X.Y branch", f("main", ["release-branch-only"]) @@ -1187,7 +1194,7 @@ def test_18e_release_only_label_refused_off_release(self): self.assertIsNone(f("release/0.192", ["release-branch-only"])) self.assertIsNone(f("main", ["fix/critical"])) - def test_18d_rbo_key_ticket_else_pr(self): + def test_rbo_key_ticket_else_pr(self): self.assertEqual(rm.rbo_key("UN-1 [FIX] x", "45"), "UN-1") self.assertEqual(rm.rbo_key("[FIX] x", "45"), "#45") self.assertEqual(rm.rbo_key(" UN-1 [FIX] x", "45"), "UN-1") @@ -1196,9 +1203,9 @@ def test_18d_rbo_key_ticket_else_pr(self): class ArtifactRegistry(unittest.TestCase): - """registry_tags over the Artifact Registry REST API. The response shape (versions[].name, createTime, - relatedTags[].name, nextPageToken) follows the API reference for projects.locations.repositories.packages - .versions.list with view=FULL; it was not recorded from a live call. + """registry_tags over the Artifact Registry REST API. + + Known limit: the response shape follows the versions.list (view=FULL) API reference, not a recorded call. """ REPO = "projects/p/locations/us-central1/repositories/unstract" @@ -1244,7 +1251,7 @@ def version(self, package, digest, created, *tags): def url(self, package, kind="unstract"): return f"{rs.AR_API}/projects/p/locations/us-central1/repositories/{kind}/packages/{package}/versions" - def test_38_tags_across_pages(self): + def test_tags_across_pages(self): self.pages[(self.url("backend"), "")] = { "versions": [ self.version( @@ -1276,7 +1283,7 @@ def test_38_tags_across_pages(self): ) # an untagged version adds nothing self.assertIn("view=FULL", self.urls[0]) - def test_39_404_is_empty_only_when_repository_reads(self): + def test_404_is_empty_only_when_repository_reads(self): onprem = "projects/p/locations/us-central1/repositories/unstract-on-prem" self.pages[(f"{rs.AR_API}/{onprem}", "")] = {"name": onprem} self.assertEqual(rs.registry_tags("on-prem/backend"), {}) @@ -1293,7 +1300,7 @@ def test_39_404_is_empty_only_when_repository_reads(self): with self.assertRaisesRegex(RuntimeError, "HTTP 403: denied"): rs.registry_tags("on-prem/backend") - def test_39b_network_json_and_later_page_errors_raise(self): + def test_network_json_and_later_page_errors_raise(self): self.pages[(self.url("backend"), "")] = {"versions": [], "nextPageToken": "n2"} with self.assertRaisesRegex(RuntimeError, "HTTP 404 on page n2"): # page n2: 404 rs.registry_tags("cloud/backend") @@ -1320,7 +1327,7 @@ def test_39b_network_json_and_later_page_errors_raise(self): ): rs.registry_tags("cloud/backend") - def test_40_other_http_errors_raise(self): + def test_other_http_errors_raise(self): self.pages[(self.url("backend"), "")] = 403 with self.assertRaisesRegex(RuntimeError, "HTTP 403: denied"): rs.registry_tags("cloud/backend") @@ -1334,7 +1341,7 @@ def test_40_other_http_errors_raise(self): ): # the error body itself cut off rs.registry_tags("cloud/backend") - def test_41_unset_location_or_token_raises(self): + def test_unset_location_or_token_raises(self): os.environ.pop("RELSTATE_AR_CHART", None) with self.assertRaisesRegex(RuntimeError, "set RELSTATE_AR_CHART"): rs.registry_tags("chart/unstract-platform") @@ -1344,5 +1351,61 @@ def test_41_unset_location_or_token_raises(self): self.assertEqual(self.urls, []) +class Guards(unittest.TestCase): + OUTSIDE = ( + "/nonexistent/repo" # resolves outside the working directory and the temp dirs + ) + + def test_option_free(self): + self.assertEqual(rm.option_free("release/1.2"), "release/1.2") + with self.assertRaises(rm.Refused): + rm.option_free("--upload-pack=x") + + def test_local_path(self): + with tempfile.TemporaryDirectory() as d: + self.assertEqual(rm.local_path(d), d) + os.symlink("/usr", os.path.join(d, "out")) + with self.assertRaises(rm.Refused): # a link out of the roots + rm.local_path(os.path.join(d, "out", "x")) + for path in (self.OUTSIDE, "-C"): + with self.subTest(path=path), self.assertRaises(rm.Refused): + rm.local_path(path) + + def test_local_path_runner_roots(self): + """GITHUB_WORKSPACE, RUNNER_TEMP and the step summary file count, each outside the other roots.""" + with tempfile.TemporaryDirectory() as d: + summary = os.path.join(d, "summary", "step.md") + for key, path in ( + ("GITHUB_WORKSPACE", d), + ("RUNNER_TEMP", d), + ("GITHUB_STEP_SUMMARY", summary), + ): + env = dict.fromkeys( + ("GITHUB_WORKSPACE", "RUNNER_TEMP", "GITHUB_STEP_SUMMARY"), "" + ) + env[key] = path + with ( + self.subTest(key=key), + unittest.mock.patch.dict(os.environ, env), + unittest.mock.patch.object( + rm.os, "getcwd", return_value="/nonexistent/cwd" + ), + unittest.mock.patch.object( + rm.tempfile, "gettempdir", return_value="/nonexistent/tmp" + ), + ): + self.assertEqual(rm.local_path(path), path) + with self.assertRaises(rm.Refused): + rm.local_path(self.OUTSIDE) + + def test_git_refuses_a_repo_outside(self): + with self.assertRaises(rm.Refused): + rm.git(self.OUTSIDE, "status") + + def test_main_refuses_an_option_as_rc(self): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + rs.main(["--repo", self.OUTSIDE, "--rc=-x"], registry=lambda a: {}) + + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 438732521c..8992a739c3 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -5,26 +5,17 @@ run-name: >- && format('record #{0}', github.event.pull_request.number) || format('picks {0}', inputs.commits || github.sha) }} -# Two jobs, one per trigger family. -# pick: every fix merged to main is cherry-picked with -x onto each stabilizing release branch. -# Each push to main rescans every target since its cut (backport.py --rescan) and picks whatever no pick or -# decline trailer names there, so a push run that a newer push replaces while pending costs nothing. -# A dispatch with `commits` picks only those; a rescan never picks a held fix, so a push never replaces a dispatch. Every pick is its own PR with the original PR's title; -# the bot squash-merges clean ones in the same run, conflicts stay open for a person. -# Security/critical fixes reach stabilized branches only with confirm_stabilized. Never builds. -# record: people land a backport conflict PR with the normal buttons (resolve, one approving review, Squash and merge) -# or close it. The bot records what happened (backport_merge.py --pr N): -# merged -> label backport-merged, head deleted; closed unmerged -> a decline: empty decline commit on the base -# (the closer's close comment is the reason), label backport-declined, head kept so the PR can be reopened; -# a head with no commit past the base declines and deletes nothing: the bot comments once that a person deletes the -# head to have the fix picked again, or keeps it until a hand-made decline is on the base; -# reopened -> label backport-conflict. Reopen and merge undoes a decline: the newest trailer on the line wins. -# pull_request_target, not pull_request: GitHub runs no pull_request workflow on a PR that conflicts with its base -# at the git level (no merge ref), and a conflict PR whose base later changed the same lines is one. -# pull_request_target runs the base line's copy of this file, so it fires only on lines that carry it. Safe: -# the job checks out main and never runs head code; backport_merge.py refuses a head that is not -# backport/X.Y/ into release/X.Y (an expression cannot match that pattern, so the job filter is coarser). A failed record run is recovered with "Re-run jobs". -# Scripts run from main's tree. +# pick (push to main, dispatch): cherry-pick -x each fix onto every stabilizing release branch, one PR per pick; +# the bot squash-merges clean picks, conflicts stay open. A push rescans every line since its cut; a dispatch +# picks only `commits`. Security/critical fixes reach stabilized lines only with confirm_stabilized. Never builds. +# record (pull_request_target closed/reopened): merged -> backport-merged, head deleted; closed -> empty decline +# commit on the base (close comment = reason), backport-declined, head kept; reopened -> backport-conflict. +# The newest trailer wins. +# Closed with no commit past the base -> nothing recorded; the bot comments once (delete the head to re-pick, +# or decline by hand). +# pull_request_target because GitHub runs no pull_request workflow on a PR that conflicts with its base. It runs +# the base line's copy of this file, checks out main and never runs head code; backport_merge.py refuses heads +# other than backport/X.Y/. A failed record run is recovered with "Re-run jobs". on: push: branches: [main] @@ -54,14 +45,13 @@ jobs: permissions: contents: read concurrency: - # Push runs are serialised, and a newer push replaces a pending one: the next rescan picks what it would have. - # A dispatch is its own group, never replaced: it may carry confirm_stabilized, which no rescan has. It can run - # beside a push run or another dispatch. Two runs picking one fix onto one line race on its pick head: the - # loser's push is rejected (retried; skipped if the winner already landed the pick, else a failed row), and a - # head with no open PR yet reads as pending until it is older than timeout-minutes. If the winner merged and - # deleted the head first, the loser can open a second pick PR for a fix already on the line. A squash after - # the base moved is retried by backport_merge.squash. + # Push runs share one group (a newer push replaces a pending one; the next rescan catches up). Each dispatch + # has its own group, since it may carry confirm_stabilized. + # Known limit: concurrent runs race on a pick head; the loser retries or skips, and can open a duplicate PR + # if the winner already merged and deleted the head. group: backport-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'rescan' }} + env: + PYTHONDONTWRITEBYTECODE: "1" # the conflict path runs `git add -A` steps: - name: Inputs if: github.event_name == 'workflow_dispatch' @@ -88,9 +78,11 @@ jobs: # A late push run checks out its event sha as origin/main; rescan the real tip instead. run: git fetch origin +refs/heads/main:refs/remotes/origin/main + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" # relstate uses dt.UTC (3.11+) + - name: Branch state - env: - GH_TOKEN: ${{ github.token }} run: | # Scripts run from outside the tree: the pick checks out release branches, which predate them. cp .github/release/*.py "$RUNNER_TEMP/" diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index 6924c56547..616d478517 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -1,14 +1,13 @@ name: Lint PR -# Every PR into main or a release branch carries a recognised `[TYPE]` title prefix, the ticket before it optional -# (`UN-123 [FIX] ...` or `[FIX] ...`, the same class either way), unless a fix/ label gives its class; its labels are -# at most one of fix/security, fix/critical plus an optional fix/needs-backport. -# Release branches too, because the release changelog is built from their PR titles. A branch cut before this file +# Every PR into main or a release branch carries a recognised `[TYPE]` title prefix, ticket optional +# (`ABC-123 [FIX] ...` or `[FIX] ...`), unless a fix/ label gives its class: at most one of fix/security, +# fix/critical, plus an optional fix/needs-backport. +# Release branches too: the release changelog is built from their PR titles. A branch cut before this file # existed runs no check. Not a required check until a ruleset makes it one. -# The backport class is only printed here; Backport recomputes it with pr_fix_class at pick time. -# Label `release-branch-only` marks a change that lives only on its release branch: refused on main. On a release -# branch, Release branch CI's bot commit writes `Release-branch-only: ` (or `#` without a ticket), which -# Squash and merge keeps in the message. +# The backport class is only printed here; Backport recomputes it at pick time. +# Label `release-branch-only` is refused on main; on a release branch, release-branch-only.yaml adds the +# `Release-branch-only: ` commit. # A PR from a fork is checked too: the run reads only the event payload and main's relmgr, with a read-only token. on: pull_request: @@ -23,9 +22,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - # The rule comes from main, so a PR's edit to relmgr cannot pass its own check. That guards relmgr only: a - # pull_request run uses the PR's copy of this file, so a PR can edit the workflow to pass. Closing that needs - # a ruleset-required workflow or CODEOWNERS on .github/. + # Rule read from main, so a PR cannot weaken relmgr. Known limit: a PR can still edit this workflow + # (needs a required workflow or CODEOWNERS on .github/). - uses: actions/checkout@v6 with: ref: main @@ -39,7 +37,7 @@ jobs: LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} BASE: ${{ github.base_ref }} run: | - if [ ! -f .github/release/relmgr.py ]; then # main before the release tooling lands: nothing to check against + if [ ! -f .github/release/relmgr.py ]; then # bootstrap: skip while main has no relmgr echo "::notice title=Lint PR::skipped: main has no .github/release/relmgr.py yet" exit 0 fi @@ -55,7 +53,7 @@ jobs: raise ValueError(refusal) cls = relmgr.pr_fix_class(title, labels) or "none" except ValueError as e: - msg = (f"{e}\nFix: title as `[TYPE] text` or `TICKET [TYPE] text` (e.g. `UN-123 [FIX] ...`); allowed labels are " + msg = (f"{e}\nFix: title as `[TYPE] text` or `TICKET [TYPE] text` (e.g. `ABC-123 [FIX] ...`); allowed labels are " "fix/security or fix/critical (at most one), and fix/needs-backport.") print("::error title=Lint PR::" + msg.replace("\n", "%0A")) # workflow commands end at a raw newline summary.write(f"### Lint PR: failed\n\n{msg}\n") diff --git a/.github/workflows/release-branch-ci.yaml b/.github/workflows/release-branch-ci.yaml index c4154ba720..c09dc04c07 100644 --- a/.github/workflows/release-branch-ci.yaml +++ b/.github/workflows/release-branch-ci.yaml @@ -1,29 +1,21 @@ name: Release branch CI -# Every commit after the cut must be a cherry-pick of a main commit (or its decline), release-branch-only, -# or build-generated. Anything else fails. -# On a PR into release/** this is the required check `check` over the PR's own commits, so a human change -# is checked before it lands. A backport/X.Y/ PR is judged as its squash: only its first commit (the pick, whose -# trailer the squash message keeps) is classified, so a plain resolution commit passes; conflict markers in the -# head's tree fail it. The push run stays as the backstop for bot pushes. -# A pull_request run uses the workflow file from the PR head: a branch cut before this trigger existed gets no PR -# check, and a PR can edit this file. Job `check` runs there; it holds no secret. -# A PR labelled `release-branch-only` passes with commits of any kind. On the label, job `trailer` pushes one -# empty commit `Release-branch-only: ` onto the PR head (unless a commit already has the line); Squash and -# merge keeps it in the message, and the push run checks that merged commit like any other. It holds the bot token, -# so it runs on pull_request_target: the base line's copy of this file, which a PR cannot edit. A line cut before -# that trigger existed gets no bot commit; add the line to a commit yourself there. -# Label changes on a closed PR run nothing. Job `trailer` never pushes to a head named main, release/* or backport/*. -# No push builds an RC: a security or critical pick lands and waits for the RM's Take RC build, like any other pick. +# Every commit after the cut must be a pick of a main commit (or its decline), release-branch-only, or +# build-generated. Anything else fails. +# PR into release/**: required check `check` over the PR's commits; a backport/X.Y/ PR is judged by its +# first commit (the squash keeps its trailer), and conflict markers fail it. A PR run holds no secret. +# Push: backstop for bot pushes. +# Label `release-branch-only`: the PR passes with commits of any kind; release-branch-only.yaml adds the +# `Release-branch-only: ` commit to its head. +# Known limit: a branch cut before these triggers existed gets no PR check; a PR can edit this file for its own +# pull_request run. +# Label changes on a closed PR run nothing. No push builds an RC: a security or critical pick waits for Take RC build. on: push: branches: ["release/**"] pull_request: branches: ["release/**"] types: [opened, synchronize, reopened, labeled, unlabeled] # the label changes the verdict - pull_request_target: - branches: ["release/**"] - types: [labeled] # job trailer only permissions: contents: read @@ -42,9 +34,8 @@ jobs: fetch-depth: 0 # relmgr needs full history and tags persist-credentials: false # the head is only read - # The classifier comes from main, so a PR's edit to relmgr cannot pass its own check. That guards relmgr - # only: a PR can still edit this workflow file, which its pull_request run uses, to pass `check`. Closing - # that needs a ruleset-required workflow or CODEOWNERS on .github/. + # Classifier read from main, so a PR cannot weaken relmgr. Known limit: a PR can still edit this workflow + # (needs a required workflow or CODEOWNERS on .github/). - uses: actions/checkout@v6 with: ref: main @@ -100,83 +91,6 @@ jobs: - name: Branch state (informational) if: always() continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} run: python3 .relmgr/.github/release/relstate.py | jq --arg b "$BRANCH" '.branches[] | select(.branch == $b)' - # TODO: alert on a failed push run once a release alert channel and webhook secret exist. - - # On the `release-branch-only` label: one empty bot commit carrying `Release-branch-only: ` (ticket from - # the title, else `#`), so the squash message has the line. A commit already carrying it -> nothing pushed. The push fires - # `synchronize`, so `check` runs once more. A fork head cannot take a bot push: fail and say so. - # pull_request_target: this job is the base line's copy. The head is only data here: read without a token, its - # files never imported, and the push ignores its git config. - trailer: - if: github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name == 'release-branch-only' && github.event.pull_request.state == 'open' - runs-on: ubuntu-latest - timeout-minutes: 20 - env: - HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - BASE_REF: ${{ github.base_ref }} - TITLE: ${{ github.event.pull_request.title }} - PR: ${{ github.event.pull_request.number }} - steps: - - name: Same-repo head - run: | - [ "$HEAD_REPO" = "$GITHUB_REPOSITORY" ] || { echo "::error::release-branch-only: the head is on $HEAD_REPO, a fork; the bot cannot push the \`Release-branch-only:\` commit there. Push the branch to $GITHUB_REPOSITORY and open the PR from it, or add the line to a commit yourself."; exit 1; } - case "$HEAD_REF" in - main | release/* | backport/*) echo "::error::release-branch-only: the head is $HEAD_REF; the bot pushes only to a topic branch. Add the line to a commit yourself."; exit 1 ;; - esac - - # The head is checked out without the bot token; only the push below gets it. The logic comes from main. - # Run on the head's tip SHA, so a push to the head after the label makes the bot push fail, not skip a check. - - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 # every branch, so origin/ is current - persist-credentials: false - - - uses: actions/checkout@v6 - with: - ref: main - path: .relmgr - sparse-checkout: .github/release - persist-credentials: false - - - name: Add the Release-branch-only commit - id: rbo - run: | - python3 -I - <<'PY' # isolated: a module file in the head checkout never shadows the stdlib - import os, subprocess, sys - sys.path.insert(0, ".relmgr/.github/release") - import relmgr as rm - ticket = rm.rbo_key(os.environ["TITLE"], os.environ["PR"]) - msgs = rm.git(".", "log", "--format=%B", f"origin/{os.environ['BASE_REF']}..HEAD") - if have := rm.RBO_RE.findall(msgs): - print(f"A commit on the PR already has Release-branch-only: {', '.join(sorted(set(have)))}; nothing pushed.") - sys.exit(0) - subprocess.run(["git", "-c", "core.hooksPath=/dev/null", "-c", "user.name=releases-bot", "-c", f"user.email={rm.BOT_EMAIL}", - "commit", "--allow-empty", "-m", f"Release-branch-only: {ticket}"], check=True) - print(f"Committed {rm.git('.', 'rev-parse', 'HEAD')[:12]} Release-branch-only: {ticket}") - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write("push=true\n") - PY - - # The repo's GitHub App, as in create-release.yaml: a GITHUB_TOKEN push or PR starts no workflow. - - name: Bot token - id: bot - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ vars.PUSH_TO_MAIN_APP_CLIENT_ID }} - private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }} - - - name: Push it - if: steps.rbo.outputs.push == 'true' - env: - BOT_TOKEN: ${{ steps.bot.outputs.token }} # a GITHUB_TOKEN push would not start the `check` re-run - run: | - # No hooks, and an explicit URL rather than the checkout's remote config. - helper='!f() { echo username=x-access-token; echo "password=$BOT_TOKEN"; }; f' - git -c core.hooksPath=/dev/null -c credential.helper= -c credential.helper="$helper" \ - push "https://github.com/${GITHUB_REPOSITORY}.git" "HEAD:refs/heads/$HEAD_REF" + # No alert on a failed push run: this repo has no release alert channel or webhook secret. diff --git a/.github/workflows/release-branch-only.yaml b/.github/workflows/release-branch-only.yaml new file mode 100644 index 0000000000..97c69b834c --- /dev/null +++ b/.github/workflows/release-branch-only.yaml @@ -0,0 +1,103 @@ +name: Release branch only + +# On the `release-branch-only` label on a PR into release/**: one empty bot commit carrying +# `Release-branch-only: ` (ticket from the title, else `#`) goes onto the PR head, unless a commit +# there already has the line. Squash and merge keeps it in the message, and Release branch CI's push run checks that +# merged commit like any other. The push fires `synchronize`, so Release branch CI's `check` runs once more. +# The job holds the bot token, so it runs on pull_request_target: the base line's copy of this file, which a PR cannot +# edit. A line whose copy of this file has no `trailer` job (cut before it existed) gets no bot commit; add the +# line to a commit yourself there. +# The head is only data here: read without a token, its files never imported, and the push ignores its git config. +# A fork head cannot take a bot push: job `fork-head` fails and says so, and job `trailer` runs only for a head in +# this repo. Label changes on a closed PR run nothing. The bot never pushes to a head named main, release/* or +# backport/*. +on: + pull_request_target: + branches: ["release/**"] + types: [labeled] + +permissions: + contents: read + +jobs: + fork-head: + if: github.event.label.name == 'release-branch-only' && github.event.pull_request.state == 'open' && github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Fork head + env: + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + run: | + echo "::error::release-branch-only: the head is on $HEAD_REPO, a fork; the bot cannot push the \`Release-branch-only:\` commit there. Push the branch to $GITHUB_REPOSITORY and open the PR from it, or add the line to a commit yourself." + exit 1 + + trailer: + if: github.event.label.name == 'release-branch-only' && github.event.pull_request.state == 'open' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + HEAD_REF: ${{ github.event.pull_request.head.ref }} + BASE_REF: ${{ github.base_ref }} + TITLE: ${{ github.event.pull_request.title }} + PR: ${{ github.event.pull_request.number }} + steps: + - name: Topic-branch head + run: | + case "$HEAD_REF" in + main | release/* | backport/*) echo "::error::release-branch-only: the head is $HEAD_REF; the bot pushes only to a topic branch. Add the line to a commit yourself."; exit 1 ;; + esac + + # The head is checked out without the bot token; only the push below gets it. The logic comes from main. + # Run on the head's tip SHA, so a push to the head after the label makes the bot push fail, not skip a check. + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 # every branch, so origin/ is current + persist-credentials: false + + - uses: actions/checkout@v6 + with: + ref: main + path: .relmgr + sparse-checkout: .github/release + persist-credentials: false + + - name: Add the Release-branch-only commit + id: rbo + run: | + python3 -I - <<'PY' # isolated: a module file in the head checkout never shadows the stdlib + import os, subprocess, sys + sys.path.insert(0, ".relmgr/.github/release") + import relmgr as rm + ticket = rm.rbo_key(os.environ["TITLE"], os.environ["PR"]) + msgs = rm.git(".", "log", "--format=%B", f"origin/{os.environ['BASE_REF']}..HEAD") + if have := rm.RBO_RE.findall(msgs): + print(f"A commit on the PR already has Release-branch-only: {', '.join(sorted(set(have)))}; nothing pushed.") + sys.exit(0) + subprocess.run(["git", "-c", "core.hooksPath=/dev/null", "-c", "user.name=releases-bot", "-c", f"user.email={rm.BOT_EMAIL}", + "commit", "--allow-empty", "-m", f"Release-branch-only: {ticket}"], check=True) + print(f"Committed {rm.git('.', 'rev-parse', 'HEAD')[:12]} Release-branch-only: {ticket}") + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write("push=true\n") + PY + + # The repo's GitHub App, as in create-release.yaml: a GITHUB_TOKEN push or PR starts no workflow. + - name: Bot token + id: bot + if: steps.rbo.outputs.push == 'true' + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.PUSH_TO_MAIN_APP_CLIENT_ID }} + private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }} + + - name: Push it + if: steps.rbo.outputs.push == 'true' + env: + BOT_TOKEN: ${{ steps.bot.outputs.token }} # a GITHUB_TOKEN push would not start the `check` re-run + run: | + # No hooks, and an explicit URL rather than the checkout's remote config. + # shellcheck disable=SC2016 # expanded by git when it runs the helper + helper='!f() { echo username=x-access-token; echo "password=$BOT_TOKEN"; }; f' + git -c core.hooksPath=/dev/null -c credential.helper= -c credential.helper="$helper" \ + push "https://github.com/${GITHUB_REPOSITORY}.git" "HEAD:refs/heads/$HEAD_REF" diff --git a/.github/workflows/release-tooling-tests.yaml b/.github/workflows/release-tooling-tests.yaml index 36be3f4515..e0c20dd214 100644 --- a/.github/workflows/release-tooling-tests.yaml +++ b/.github/workflows/release-tooling-tests.yaml @@ -9,6 +9,7 @@ on: - ".github/workflows/backport.yaml" - ".github/workflows/lint-pr.yaml" - ".github/workflows/release-branch-ci.yaml" + - ".github/workflows/release-branch-only.yaml" - ".github/workflows/release-tooling-tests.yaml" - ".github/workflows/state.yaml" diff --git a/.github/workflows/state.yaml b/.github/workflows/state.yaml index 9a97daca3b..6ae35254c7 100644 --- a/.github/workflows/state.yaml +++ b/.github/workflows/state.yaml @@ -21,9 +21,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - start=$(date +%s.%N) python3 .github/release/relstate.py > state.json - echo "relstate took $(echo "$(date +%s.%N) - $start" | bc) s" cat state.json - uses: actions/upload-artifact@v7 From 3dc442c988097bdfcf940bbfa6493feadf75d612 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 10:21:37 +0530 Subject: [PATCH 29/37] test: keep one call inside the symlink refusal check Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/test_relstate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index d8acd4b50b..40a0b3f480 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -1365,8 +1365,9 @@ def test_local_path(self): with tempfile.TemporaryDirectory() as d: self.assertEqual(rm.local_path(d), d) os.symlink("/usr", os.path.join(d, "out")) - with self.assertRaises(rm.Refused): # a link out of the roots - rm.local_path(os.path.join(d, "out", "x")) + escape = os.path.join(d, "out", "x") # a link out of the roots + with self.assertRaises(rm.Refused): + rm.local_path(escape) for path in (self.OUTSIDE, "-C"): with self.subTest(path=path), self.assertRaises(rm.Refused): rm.local_path(path) From f12defbe12622c9617ce4e3f0faaf451b2267f43 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 11:46:30 +0530 Subject: [PATCH 30/37] fix: drop an unused token from the state workflow Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/state.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/state.yaml b/.github/workflows/state.yaml index 6ae35254c7..78057d372c 100644 --- a/.github/workflows/state.yaml +++ b/.github/workflows/state.yaml @@ -18,8 +18,6 @@ jobs: - name: Derive branch and target state id: state - env: - GH_TOKEN: ${{ github.token }} run: | python3 .github/release/relstate.py > state.json cat state.json From ba9e8ac9979c5c30e2d54b42a273fd991cae6025 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 12:13:09 +0530 Subject: [PATCH 31/37] fix: pass workflow inputs to the release CLIs as --flag=value A dispatch input that starts with "-" now reaches the CLI as a value, so it is refused with the usual message instead of an argparse usage error. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/backport.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 8992a739c3..40c370e1b3 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -96,9 +96,9 @@ jobs: CONFIRM: ${{ inputs.confirm_stabilized }} BRANCHES: ${{ inputs.branches }} run: | - if [ "$GITHUB_EVENT_NAME" = push ]; then args=(--rescan); else args=(--commits "$COMMITS"); fi + if [ "$GITHUB_EVENT_NAME" = push ]; then args=(--rescan); else args=(--commits="$COMMITS"); fi if [ "$CONFIRM" = true ]; then args+=(--confirm-stabilized); fi - python3 "$RUNNER_TEMP/backport.py" --state "$RUNNER_TEMP/state.json" "${args[@]}" --branches "$BRANCHES" + python3 "$RUNNER_TEMP/backport.py" --state "$RUNNER_TEMP/state.json" "${args[@]}" --branches="$BRANCHES" record: if: >- From 913d15442d5b69ace80f3a2170e5ad364849ffdc Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 12:49:46 +0530 Subject: [PATCH 32/37] fix: skip Greptile review on the release bot's PRs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .greptile/config.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .greptile/config.json diff --git a/.greptile/config.json b/.greptile/config.json new file mode 100644 index 0000000000..f122de02ee --- /dev/null +++ b/.greptile/config.json @@ -0,0 +1,5 @@ +{ + "excludeAuthors": [ + "zipstack-push-to-main[bot]" + ] +} From 7f1e72e88b403f7573fde92da0ce0e476168fa28 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 13:37:36 +0530 Subject: [PATCH 33/37] Read an optional RC repository in the release state tooling relstate reads a second repository per kind when RELSTATE_AR__RC is set, and image links point at it. With the variable unset, nothing changes. The tests for the cloud-only workflow steps skip here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/relmgr.py | 7 +- .github/release/relstate.py | 120 +++--- .github/release/test_oss.py | 608 ++++++++++++++++++++++++++++++- .github/release/test_relstate.py | 140 ++++++- 4 files changed, 821 insertions(+), 54 deletions(-) diff --git a/.github/release/relmgr.py b/.github/release/relmgr.py index 2ed997e550..13460fe803 100644 --- a/.github/release/relmgr.py +++ b/.github/release/relmgr.py @@ -55,11 +55,12 @@ def tag_link(tag): def image_link(target, tag): - """The Artifact Registry console page of target's repository (RELSTATE_AR_), the tag in the text; - plain text when that variable is unset. + """The Artifact Registry console page of the repository holding target's RC builds (RELSTATE_AR__RC, + else RELSTATE_AR_), the tag in the text; plain text when neither is set. """ text = f"`{target}:{tag}`" - repo = os.environ.get("RELSTATE_AR_" + target.upper().replace("-", "_"), "") + var = "RELSTATE_AR_" + target.upper().replace("-", "_") + repo = os.environ.get(var + "_RC") or os.environ.get(var, "") m = re.fullmatch(r"projects/([^/]+)/locations/([^/]+)/repositories/([^/]+)", repo) return ( f"[{text}](https://console.cloud.google.com/artifacts/docker/{m[1]}/{m[2]}/{m[3]})" diff --git a/.github/release/relstate.py b/.github/release/relstate.py index 24bf369802..4c1e493c2c 100644 --- a/.github/release/relstate.py +++ b/.github/release/relstate.py @@ -29,8 +29,8 @@ # Per repo, so both repos run this file; use() loads an entry as (TARGETS, LATEST_ONLY: only the latest GA line is # supported, KEPT: supported while the branch exists, PINS: repo whose tags pyproject.toml pins, PREFIX: tag prefix). # TARGETS: target -> (build artifact, GA anchor artifact), each /; RELSTATE_AR_ names its -# repository. GIT: the line's own git tags stand in for an artifact (OSS lines). OSS lines are tagged -# ee-vX.Y.Z(-rc.N). +# repository and RELSTATE_AR__RC, when set, a second one holding its RC builds. GIT: the line's own git tags +# stand in for an artifact (OSS lines). OSS lines are tagged ee-vX.Y.Z(-rc.N). GIT = "git" REPOS = { # backend: every build pushes it, and all services share one tag. @@ -113,10 +113,19 @@ def _ar_get(url, what): raise RuntimeError(f"registry read {what}: {e!r}") from e -def _ar_versions(artifact): - """Every version of artifact (/) with its tags, or None when its first page answers 404.""" - kind, package = artifact.split("/", 1) - url = f"{AR_API}/{ar_repository(kind)}/packages/{urllib.parse.quote(package, safe='')}/versions" +def ar_repositories(kind): + """The repositories registry_tags reads for an artifact kind: RELSTATE_AR_ (stable builds), then + RELSTATE_AR__RC (RC builds) when it is set to another repository. With _RC unset, one repository holds + both. Both are read because Retag promotes an RC into the stable repository, which then holds the GA tag. + """ + stable = ar_repository(kind) + rc = os.environ.get("RELSTATE_AR_" + kind.upper().replace("-", "_") + "_RC") + return [stable, *([rc] if rc and rc != stable else [])] + + +def _ar_versions(repo, package, artifact): + """Every version of package in repo with its tags, or None when its first page answers 404.""" + url = f"{AR_API}/{repo}/packages/{urllib.parse.quote(package, safe='')}/versions" out, page = [], "" while True: q = {"view": "FULL", "pageSize": 1000, **({"pageToken": page} if page else {})} @@ -130,32 +139,43 @@ def _ar_versions(artifact): def registry_tags(artifact): - """{tag: {id, digest, pushed_at}} for one Artifact Registry artifact (/). Raises on any read - failure. pushed_at is the version's createTime, to the second. + """{tag: {id, digest, pushed_at}} for one Artifact Registry artifact (/), over every repository + of its kind (ar_repositories). Raises on any read failure. pushed_at is the version's createTime, to the second. A 404 is only read as "artifact not pushed yet" when the artifact's own repository reads; otherwise it may be - a wrong RELSTATE_AR_, and an empty registry would read as "nothing enabled". + a wrong RELSTATE_AR_, and an empty registry would read as "nothing enabled". A tag held by two + repositories at different digests raises: the state would depend on which one was read. That stops every + reader, for every line and Retire included, until a person deletes one copy: which one was tested is not + something to guess. A tag held by both at one digest reads as its first repository's copy only. """ - versions = _ar_versions(artifact) - if versions is None: - repo = ar_repository(artifact.split("/", 1)[0]) - if _ar_get(f"{AR_API}/{repo}", artifact) is None: - raise RuntimeError( - f"registry read {artifact}: 404 and repository {repo} not found" - ) - return {} + kind, package = artifact.split("/", 1) out = {} - for v in versions: - pushed = re.sub( - r"\.\d+(?=Z$)", "", v["createTime"] - ) # AR returns up to 9 fraction digits - for t in v.get("relatedTags") or []: - tag = t["name"].rsplit("/tags/", 1)[1] - out[tag] = { - "id": v["name"], - "digest": v["name"].rsplit("/versions/", 1)[1], - "pushed_at": pushed, - } + for repo in ar_repositories(kind): + versions = _ar_versions(repo, package, artifact) + if versions is None: + if _ar_get(f"{AR_API}/{repo}", artifact) is None: + raise RuntimeError( + f"registry read {artifact}: 404 and repository {repo} not found" + ) + continue + for v in versions: + pushed = re.sub( + r"\.\d+(?=Z$)", "", v["createTime"] + ) # AR returns up to 9 fraction digits + for t in v.get("relatedTags") or []: + tag = t["name"].rsplit("/tags/", 1)[1] + info = { + "id": v["name"], + "digest": v["name"].rsplit("/versions/", 1)[1], + "pushed_at": pushed, + } + if tag in out and out[tag]["digest"] != info["digest"]: + raise RuntimeError( + f"registry read {artifact}: tag {tag} is {out[tag]['id']} (pushed {out[tag]['pushed_at']})" + f" and {info['id']} (pushed {pushed}); Retag promotes the RC repository's copy. Check" + " which one was tested, delete the other (Artifact Registry console) and rerun" + ) + out.setdefault(tag, info) return out @@ -533,33 +553,44 @@ def _branch_row(state, branch): return next((b for b in state["branches"] if b["branch"] == branch), None) -def _by_digest(tags): - """{digest: {id, digest, tags}} of an artifact's {tag: info}.""" - by_digest = {} +def _repo_of(version): + """The repository of an abandon_plan version (its id up to /packages/); "" for a version with no id.""" + return (version["id"] or "").split("/packages/", 1)[0] + + +def _by_version(tags): + """{id or digest: {id, digest, tags}} of an artifact's {tag: info}. One digest in two repositories is two + versions. + """ + by_version = {} # Version order, so the last version is the newest RC (rc.10 after rc.9); other tags first. for t, info in sorted( tags.items(), key=lambda kv: (_vkey(kv[0]) if VER_RE.fullmatch(kv[0]) else (-1, -1), kv[0]), ): - by_digest.setdefault( - info["digest"], + by_version.setdefault( + info.get("id") or info["digest"], {"id": info.get("id"), "digest": info["digest"], "tags": []}, )["tags"].append(t) - return by_digest + return by_version def _abandon_versions(reg, rc_re): - """abandon_plan's versions to delete per artifact and the last versions kept.""" + """abandon_plan's versions to delete per artifact and the last versions kept. Per repository, when every + tagged version would go, the last is kept, so no package is emptied. + Known limits: untagged versions are not listed, so "last" counts tagged ones only; one tag at one digest in + both repositories reads as the stable copy only, so the RC repository's copy is not deleted. + """ versions, kept = {}, [] for a, tags in reg.items(): - by_digest = _by_digest(tags) - drop = [ - v for v in by_digest.values() if all(rc_re.fullmatch(t) for t in v["tags"]) - ] - # Known limit: untagged versions are not listed, so "last" counts tagged ones only. - if drop and len(drop) == len(by_digest): - kept.append(f"{a}:{','.join(drop[-1]['tags'])}") - drop = drop[:-1] + by_version, drop = _by_version(tags), [] + for where in sorted({_repo_of(v) for v in by_version.values()}): + mine = [v for v in by_version.values() if _repo_of(v) == where] + d = [v for v in mine if all(rc_re.fullmatch(t) for t in v["tags"])] + if d and len(d) == len(mine): + kept.append(f"{a}:{','.join(d[-1]['tags'])}") + d = d[:-1] + drop += d if drop: versions[a] = drop return versions, kept @@ -571,7 +602,8 @@ def abandon_plan(repo, branch, registry=registry_tags): {"ga": [...], "used": [...], "rc_tags": [...], "versions": {artifact: [{id, digest, tags}]}, "kept": [...]}. A non-empty ga (git stable tag or GA-anchor registry tag) or used (commits that are not bot, pick or decline) refuses the abandon. A registry version is deleted only when every tag on it is an RC of - this line. An artifact's last tagged version is kept and listed, so an abandon never empties an artifact. + this line. When that would delete every tagged version in a repository, the last one is kept and listed, + so an abandon never empties a repository's package. """ x, y = rm._line(branch) rc_re = re.compile(rf"{re.escape(PREFIX)}v{x}\.{y}\.\d+-rc\.\d+") diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 8b4314b589..c5c0894ea4 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -680,6 +680,29 @@ def inline(workflow, step): ) +def step(workflow, name, exprs=()): + """The bash run block of a workflow step, each ${{ }} expression replaced from exprs ("" when absent).""" + with open(os.path.join(HERE, "..", "workflows", workflow)) as fh: + lines = fh.read().split("\n") + i = next( + n + for n, line in enumerate(lines) + if line.strip() in (f"- name: {name}", f"name: {name}") + ) + i = next(n for n in range(i, len(lines)) if lines[n].strip() == "run: |") + indent = len(lines[i]) - len(lines[i].lstrip()) + 2 + body = [] + for line in lines[i + 1 :]: + if line.strip() and len(line) - len(line.lstrip()) < indent: + break + body.append(line[indent:]) + return re.sub( + r"\$\{\{\s*(.*?)\s*\}\}", + lambda m: dict(exprs).get(m.group(1), ""), + "\n".join(body), + ) + + @unittest.skipUnless( os.path.exists(os.path.join(HERE, "..", "workflows", "retire.yaml")), "the cloud repo's workflows", @@ -710,15 +733,23 @@ def setUp(self): fh.write(f"""import json, os, sys sys.path.insert(0, {HERE!r}) import relstate -def _ar_versions(artifact): # reg.jsonl for cloud/backend, reg-_.jsonl for the others +def _ar_versions(repo, package, artifact): # reg.jsonl for cloud/backend, reg-_.jsonl for the others; a row's "repo" (default r) is its repository, "names" its version ids' repository f = os.path.join({d!r}, "reg.jsonl" if artifact == "cloud/backend" else "reg-" + artifact.replace("/", "_") + ".jsonl") if not os.path.exists(f): return [] with open(f) as fh: - rows = [json.loads(line) for line in fh] - return [{{"name": f"r/packages/backend/versions/{{v['digest']}}", "createTime": v["pushed_at"], - "relatedTags": [{{"name": f"r/packages/backend/tags/{{t}}"}} for t in v["tags"]]}} for v in rows] + rows = [v for v in map(json.loads, fh) if v.get("repo", "r") == repo] + return [{{"name": f"{{v.get('names', repo)}}/packages/backend/versions/{{v['digest']}}", "createTime": v["pushed_at"], + "relatedTags": [{{"name": f"{{repo}}/packages/backend/tags/{{t}}"}} for t in v["tags"]]}} for v in rows] relstate._ar_versions = _ar_versions +for kind in ("CLOUD", "ON_PREM", "CHART"): + os.environ.setdefault("RELSTATE_AR_" + kind, "r") +import urllib.request +def _urlopen(req, timeout=None): # a registry delete, logged to ar.log + with open({os.path.join(d, "ar.log")!r}, "a") as fh: + fh.write(f"{{req.get_method()}} {{req.full_url}}\\n") + return open(os.devnull, "rb") +urllib.request.urlopen = _urlopen """) bin_ = os.path.join(d, "bin") os.mkdir(bin_) @@ -817,6 +848,54 @@ def test_abandon(self): self.retire(ABANDON="release/1.7", REASON="cut by mistake") self.assert_gone("1.7") + def test_abandon_deletes_one_rc_and_keeps_the_last(self): + self.line("1.7", "v1.7.0-rc.1", "v1.7.0-rc.2") + self.retire(ABANDON="release/1.7", REASON="cut by mistake", RELSTATE_AR_TOKEN="t") + self.assertRegex( + self.read("ar.log"), + r"^DELETE \S+/r/packages/backend/versions/sha256:v1\.7\.0-rc\.1\?force=true\n$", + ) + + def test_abandon_deletes_in_the_rc_repository(self): + self.line("1.7") + for i in (1, 2): # RCs built after the split: only in the RC repository + with open(os.path.join(self.d, "reg.jsonl"), "a") as fh: + fh.write( + f'{{"repo": "rc", "digest": "sha256:rc{i}", "pushed_at": "2026-09-01T00:00:00Z", ' + f'"tags": ["v1.7.0-rc.{i}"]}}\n' + ) + self.retire( + ABANDON="release/1.7", + REASON="cut by mistake", + RELSTATE_AR_TOKEN="t", + RELSTATE_AR_CLOUD_RC="rc", + ) + self.assertRegex( + self.read("ar.log"), + r"^DELETE \S+/rc/packages/backend/versions/sha256:rc1\?force=true\n$", + ) + + def test_abandon_refuses_a_version_outside_its_repositories(self): + self.line("1.7") + for i in (1, 2): # read from repository r, but their ids name repository other + with open(os.path.join(self.d, "reg.jsonl"), "a") as fh: + fh.write( + f'{{"names": "other", "digest": "sha256:rc{i}", "pushed_at": "2026-09-01T00:00:00Z", ' + f'"tags": ["v1.7.0-rc.{i}"]}}\n' + ) + r = self.run_step( + inline("retire.yaml", "Derive and retire"), + ABANDON="release/1.7", + REASON="cut by mistake", + RETIRE_LINE="", + NOW="", + DRY_RUN="false", + RELSTATE_AR_TOKEN="t", + ) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("other/packages/backend/versions/sha256:rc1 is outside r", r.stdout) + self.assertFalse(os.path.exists(os.path.join(self.d, "ar.log"))) + def test_retire_line_no_ga(self): self.line("1.7") self.retire(RETIRE_LINE="release/1.7", REASON="not shipping") @@ -928,6 +1007,527 @@ def test_derived_list_refuses_a_line_moved_after_its_archive(self): self.refused("1.6", tag) +@unittest.skipUnless( + os.path.exists(os.path.join(HERE, "..", "workflows", "sync-argocd.yaml")), + "the cloud repo's workflows", +) +class SyncArgocdParameters(unittest.TestCase): + """The helm.parameters sync-argocd writes back: its own keys replaced, every other parameter on the app kept.""" + + OTHER = { + "name": "postgresql.enabled", + "value": "false", + } # a chart value this workflow does not own + REG = "l-docker.pkg.dev/p/rc" + + def merge(self, current, tag="", cloud_rc="", revision="main"): + return subprocess.run( + [ + sys.executable, + "-c", + inline("sync-argocd.yaml", "Update ArgoCD Application"), + ], + env={ + **os.environ, + "CURRENT": json.dumps(current), + "IMAGE_TAG": tag, + "CLOUD_RC": cloud_rc, + "TARGET_REVISION": revision, + "APP_NAME": "dev-unstract-dev", + }, + capture_output=True, + text=True, + ) + + def merged(self, current, *args, **kwargs): + r = self.merge(current, *args, **kwargs) + self.assertEqual(r.returncode, 0, r.stderr) + params = json.loads(r.stdout) + self.assertEqual( + len(params), len({p["name"] for p in params}), params + ) # each name once + return {p["name"]: p["value"] for p in params} + + def test_rc_sync_adds_the_registries(self): + got = self.merged( + [self.OTHER, {"name": "global.unstract.image.tag", "value": "v1"}], + "v2.0.0-rc.1", + self.REG, + ) + self.assertEqual(got["postgresql.enabled"], "false") + self.assertEqual(got["global.unstract.image.tag"], "v2.0.0-rc.1") + self.assertEqual(got["backend.image.registry"], self.REG) + self.assertEqual( + got["runner.configMap.TOOL_SIDECAR_IMAGE_NAME"], f"{self.REG}/tool-sidecar" + ) + self.assertEqual(got["runner.configMap.TOOL_SIDECAR_IMAGE_TAG"], "v2.0.0-rc.1") + + def test_non_rc_sync_drops_the_registries(self): + rc = [ + {"name": n, "value": v} + for n, v in self.merged([self.OTHER], "v2.0.0-rc.1", self.REG).items() + ] + self.assertEqual( + self.merged(rc, "v2", self.REG), + {"postgresql.enabled": "false", "global.unstract.image.tag": "v2"}, + ) + + def test_no_tag_keeps_the_current_one(self): + self.assertEqual( + self.merged( + [self.OTHER, {"name": "global.unstract.image.tag", "value": "v1"}] + ), + {"postgresql.enabled": "false", "global.unstract.image.tag": "v1"}, + ) + + def test_a_kept_rc_tag_pulls_from_cloud_rc(self): + # a re-sync with no tag keeps the app's RC tag, and rewrites its registries to the RC repository + tag = {"name": "global.unstract.image.tag", "value": "v2.0.0-rc.1"} + stable = "us-central1-docker.pkg.dev/p/unstract" + got = self.merged( + [self.OTHER, tag, {"name": "backend.image.registry", "value": stable}], + cloud_rc=self.REG, + ) + self.assertEqual(got["global.unstract.image.tag"], "v2.0.0-rc.1") + self.assertEqual(got["backend.image.registry"], self.REG) + self.assertEqual(got["frontend.image.registry"], self.REG) + self.assertEqual(got["runner.configMap.TOOL_SIDECAR_IMAGE_TAG"], "v2.0.0-rc.1") + + def test_an_rc_revision_pulls_from_cloud_rc(self): + # no tag anywhere: the chart's own tag at an RC revision is that RC + got = self.merged([self.OTHER], cloud_rc=self.REG, revision="v1.1.0-rc.2") + self.assertEqual(got["backend.image.registry"], self.REG) + self.assertNotIn("global.unstract.image.tag", got) + self.assertNotIn("runner.configMap.TOOL_SIDECAR_IMAGE_TAG", got) + + def test_an_rc_tag_without_cloud_rc_fails(self): + tag = {"name": "global.unstract.image.tag", "value": "v2.0.0-rc.1"} + for current, kwargs in ( + ([self.OTHER], {"tag": "v2.0.0-rc.1"}), + ([self.OTHER, tag], {}), + ([self.OTHER], {"revision": "v1.1.0-rc.2"}), + ): + r = self.merge(current, **kwargs) + self.assertEqual(r.returncode, 1, kwargs) + self.assertIn("RELSTATE_AR_CLOUD_RC, which is unset", r.stderr, kwargs) + self.assertEqual(r.stdout, "", kwargs) + + def test_the_sidecar_tag_follows_a_tag_parameter(self): + # no tag parameter: the chart's tag and sidecar tag are the same anchor + got = self.merged( + [ + self.OTHER, + {"name": "runner.configMap.TOOL_SIDECAR_IMAGE_TAG", "value": "x"}, + ], + cloud_rc=self.REG, + revision="v1.1.0-rc.2", + ) + self.assertNotIn("runner.configMap.TOOL_SIDECAR_IMAGE_TAG", got) + self.assertEqual( + self.merged([self.OTHER], "snapshot.7"), + {"postgresql.enabled": "false", "global.unstract.image.tag": "snapshot.7"}, + ) + + def test_cloud_rc_is_not_taken_without_an_rc_tag(self): + for tag in ("snapshot.7", "v2.0.0", "rc.3"): + for cloud_rc in (self.REG, ""): + self.assertEqual( + self.merged([self.OTHER], tag, cloud_rc), + {"postgresql.enabled": "false", "global.unstract.image.tag": tag}, + ) + + +RC = "projects/p/locations/l/repositories/rc" +RC_HOST = "l-docker.pkg.dev/p/rc" + + +@unittest.skipUnless( + os.path.exists(os.path.join(HERE, "..", "workflows", "build-push.yaml")), + "the cloud repo's workflows", +) +class GuardSteps(unittest.TestCase): + """Workflow steps run as bash from the workflow files: the input guards of build-push and dev-deploy, sync-argocd's + update step, retag's copy and the build workflows' image roots. Stubs: curl (the GET answers get.json with the + HTTP code in code, the PATCH body lands in patch.json); crane (img/ holds an image's digest, a missing one + prints crane_err and fails, each copy is logged to copies, drift makes a copy land at another digest); docker + (login only). + """ + + APP = { + "spec": { + "source": { + "helm": {"parameters": [{"name": "postgresql.enabled", "value": "false"}]} + } + } + } + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.d = d = tmp.name + with open(os.path.join(d, "curl"), "w") as fh: + fh.write(f"""#!/bin/sh +case "$*" in + *"-X PATCH"*) cat > {d}/patch.json; printf '{{}}\\n200';; + *) cat {d}/get.json; printf '\\n%s' "$(cat {d}/code)";; +esac +""") + os.chmod(os.path.join(d, "curl"), 0o755) + os.mkdir(os.path.join(d, "img")) + with open(os.path.join(d, "crane"), "w") as fh: + fh.write(f"""#!/bin/sh +img() {{ echo "{d}/img/$(echo "$1" | tr '/:' '__')"; }} +case "$1" in + auth) cat > /dev/null;; + digest) [ -e "$(img "$2")" ] || {{ cat {d}/crane_err; exit 1; }}; cat "$(img "$2")";; + copy) echo "$2 $3" >> {d}/copies; cp "$(img "$2")" "$(img "$3")"; [ ! -e {d}/drift ] || echo sha256:other > "$(img "$3")";; +esac +""") + with open(os.path.join(d, "docker"), "w") as fh: + fh.write("#!/bin/sh\ncat > /dev/null\n") + for tool in ("crane", "docker"): + os.chmod(os.path.join(d, tool), 0o755) + + def bash(self, script, **env): + return subprocess.run( + ["bash", "-eo", "pipefail", "-c", script], + env={ + "PATH": f"{self.d}:{os.environ['PATH']}", + "GITHUB_OUTPUT": os.path.join(self.d, "out"), + "GITHUB_STEP_SUMMARY": os.path.join(self.d, "summary"), + "GITHUB_ENV": os.path.join(self.d, "env"), + **env, + }, + capture_output=True, + text=True, + ) + + def build_push(self, tag, **env): + script = step( + "build-push.yaml", + "Assert tags when manual", + {"github.event.inputs.tag || inputs.tag": tag}, + ) + return self.bash( + script, + **{ + "BUILD_FROM_TAG": "false", + "BUILDS": "true", + "RC_REPOS": "projects/p/locations/l/repositories/rc", + **env, + }, + ) + + def test_build_push_refuses_an_rc_tag_built_outside_build_from_tag(self): + r = self.build_push("v1.2.0-rc.1") + self.assertEqual(r.returncode, 1, r.stdout) + self.assertIn("release-line RC tag", r.stdout) + + def test_build_push_passes_an_rc_tag_otherwise(self): + for env in ({"BUILD_FROM_TAG": "true"}, {"BUILDS": "false"}, {"RC_REPOS": ""}): + r = self.build_push("v1.2.0-rc.1", **env) + self.assertEqual(r.returncode, 0, (env, r.stdout)) + for tag in ("snapshot.7", "v1.2.0", "rc.4"): + self.assertEqual(self.build_push(tag).returncode, 0, tag) + + def test_dev_deploy_refuses_frontend_dev_for_an_rc_tag(self): + guard = step("dev-deploy.yaml", "Refuse frontend-dev for an RC tag") + env = {"TAG": "v1.2.0-rc.1", "FRONTEND_DEV": "true", "CLOUD_RC": RC} + r = self.bash(guard, **env) + self.assertEqual(r.returncode, 1, r.stdout) + for change in ({"TAG": "auto"}, {"FRONTEND_DEV": "false"}, {"CLOUD_RC": ""}): + self.assertEqual(self.bash(guard, **{**env, **change}).returncode, 0, change) + + def sync(self, app, code=200, **env): + with open(os.path.join(self.d, "get.json"), "w") as fh: + fh.write(app if isinstance(app, str) else json.dumps(app)) + with open(os.path.join(self.d, "code"), "w") as fh: + fh.write(str(code)) + r = self.bash( + step("sync-argocd.yaml", "Update ArgoCD Application"), + **{ + "APP_NAME": "dev-unstract-dev", + "TARGET_REVISION": "main", + "IMAGE_TAG": "", + "CLOUD_RC": "", + "TRIGGERED_BY": "dev", + "ARGOCD_SERVER": "argocd", + "ARGOCD_TOKEN": "t", + **env, + }, + ) + patch = os.path.join(self.d, "patch.json") + if not os.path.exists(patch): + return r, None + with open(patch) as fh: + params = json.loads(json.load(fh)["patch"])["spec"]["source"]["helm"][ + "parameters" + ] + os.remove(patch) + return r, {p["name"]: p["value"] for p in params} + + def test_sync_refuses_a_read_with_no_spec_source(self): + for body in ( + '{"error": "permission denied"}', + '{"spec": {}}', + "login", + "", + ): + r, params = self.sync(body) + self.assertEqual(r.returncode, 1, (body, r.stdout)) + self.assertIn( + "returned no .spec.source", r.stdout, body + ) # named, not a traceback + self.assertIsNone(params, body) + r, params = self.sync(self.APP, code=403) + self.assertEqual((r.returncode, params), (1, None)) + + def test_sync_of_an_app_with_no_parameters(self): + r, params = self.sync({"spec": {"source": {}}}, IMAGE_TAG="snapshot.7") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual(params, {"global.unstract.image.tag": "snapshot.7"}) + + def test_sync_keeps_an_unrelated_parameter_and_takes_cloud_rc(self): + r, params = self.sync(self.APP, IMAGE_TAG="v1.2.0-rc.1", CLOUD_RC=RC) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual(params["postgresql.enabled"], "false") + self.assertEqual(params["backend.image.registry"], RC_HOST) + with open(os.path.join(self.d, "env")) as fh: + self.assertIn(f"IMAGE_REGISTRY_APPLIED={RC_HOST}", fh.read()) + r, params = self.sync(self.APP, IMAGE_TAG="snapshot.7", CLOUD_RC=RC) + self.assertEqual( + params, + {"postgresql.enabled": "false", "global.unstract.image.tag": "snapshot.7"}, + ) + + def app(self, revision="", **params): + return { + "spec": { + "source": { + "targetRevision": revision, + "helm": { + "parameters": [{"name": n, "value": v} for n, v in params.items()] + }, + } + } + } + + def test_resync_of_an_rc_pulls_from_cloud_rc(self): + stable = "us-central1-docker.pkg.dev/p/unstract" + regs = { + f"{s}.image.registry": stable + for s in ("frontend", "backend", "runner", "platform", "x2text", "worker") + } + regs["runner.configMap.TOOL_SIDECAR_IMAGE_NAME"] = f"{stable}/tool-sidecar" + rc_regs = {k: RC_HOST for k in regs if k.endswith("registry")} + rc_regs["runner.configMap.TOOL_SIDECAR_IMAGE_NAME"] = f"{RC_HOST}/tool-sidecar" + # a dev namespace on an RC tag, re-synced with no tag on any branch: its registries become the RC repository + app = self.app("main", **{"global.unstract.image.tag": "v1.1.0-rc.2", **regs}) + for revision in ("main", "feature/x"): + r, params = self.sync(app, TARGET_REVISION=revision, CLOUD_RC=RC) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + params, + { + "global.unstract.image.tag": "v1.1.0-rc.2", + "runner.configMap.TOOL_SIDECAR_IMAGE_TAG": "v1.1.0-rc.2", + **rc_regs, + }, + ) + # an app after a take-rc-build sync: registries and no tag parameter, the chart at the RC git tag + for current in ( + self.app("v1.2.0-rc.1", **rc_regs), + self.app("v1.1.0-rc.2", **regs), + ): + r, params = self.sync( + current, + APP_NAME="a-unstract-dev", + TARGET_REVISION="v1.2.0-rc.1", + CLOUD_RC=RC, + ) + self.assertEqual(params, rc_regs) + + def test_an_rc_sync_without_cloud_rc_fails(self): + for app, env in ( + (self.APP, {"IMAGE_TAG": "v1.2.0-rc.1"}), + (self.APP, {"TARGET_REVISION": "v1.2.0-rc.1"}), + (self.app("main", **{"global.unstract.image.tag": "v1.1.0-rc.2"}), {}), + ): + r, params = self.sync(app, **env) + self.assertEqual((r.returncode, params), (1, None), env) + self.assertIn("RELSTATE_AR_CLOUD_RC, which is unset", r.stderr, env) + # a non-RC sync drops RC registries an app still has, with the variable unset + r, params = self.sync( + self.app("v1.2.0-rc.1", **{"backend.image.registry": RC_HOST}), + TARGET_REVISION="v1.3.0", + ) + self.assertEqual((r.returncode, params), (0, {})) + + def test_rc_target_revision_with_no_tag_takes_cloud_rc(self): + _, params = self.sync(self.APP, TARGET_REVISION="v1.2.0-rc.1", CLOUD_RC=RC) + self.assertEqual(params["backend.image.registry"], RC_HOST) + _, params = self.sync(self.APP, TARGET_REVISION="feature/x", CLOUD_RC=RC) + self.assertEqual(params, {"postgresql.enabled": "false"}) + + def test_sync_refuses_a_malformed_repository(self): + # a newline would add a line to GITHUB_ENV + for bad in ("bogus", "projects/p\nX=1/locations/l/repositories/r"): + for tag in ("v1.2.0-rc.1", "snapshot.7"): + r, params = self.sync(self.APP, IMAGE_TAG=tag, CLOUD_RC=bad) + self.assertEqual((r.returncode, params), (1, None), (bad, tag)) + self.assertIn("Not an AR repository", r.stderr, bad) + + STABLE = "us-central1-docker.pkg.dev/p/unstract" + + def retag(self, images, err="", drift=False, rc_repo=RC, source="v1.2.0-rc.1"): + for image, digest in images.items(): + with open( + os.path.join(self.d, "img", image.replace("/", "_").replace(":", "_")), + "w", + ) as fh: + fh.write(digest) + with open(os.path.join(self.d, "crane_err"), "w") as fh: + fh.write(err) + if drift: + open(os.path.join(self.d, "drift"), "w").close() + r = self.bash( + step("retag-docker-images.yaml", "Retag ${{ matrix.service }} image"), + PROJECT_ID="p", + SOURCE_TAG=source, + TARGET_TAG="v1.2.0", + SERVICE="backend", + RC_REPO=rc_repo, + TOKEN="t", + ) + copies = os.path.join(self.d, "copies") + with open(copies) if os.path.exists(copies) else io.StringIO() as fh: + return r, fh.read().split("\n")[0] + + def test_retag_copies_the_rc_repository_image(self): + r, copy = self.retag( + { + f"{RC_HOST}/backend:v1.2.0-rc.1": "sha256:rc", + f"{self.STABLE}/backend:v1.2.0-rc.1": "sha256:old", + } + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + copy, f"{RC_HOST}/backend:v1.2.0-rc.1 {self.STABLE}/backend:v1.2.0" + ) + + def test_retag_reads_a_release_line_rc_only_from_its_rc_repository(self): + stable = {f"{self.STABLE}/backend:v1.2.0-rc.1": "sha256:old"} + for rc_repo in ("", "bogus", "projects/p/locations/l/repositories/r/x"): + r, copy = self.retag(stable, rc_repo=rc_repo) + self.assertEqual((r.returncode, copy), (1, ""), rc_repo) + self.assertIn("is a release-line RC", r.stdout, rc_repo) + for source in ("rc.4", "snapshot.7"): + r, copy = self.retag( + {f"{self.STABLE}/backend:{source}": "sha256:s"}, source=source + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual( + copy, f"{self.STABLE}/backend:{source} {self.STABLE}/backend:v1.2.0" + ) + os.remove(os.path.join(self.d, "copies")) + + def test_retag_refuses_a_copy_at_another_digest(self): + r, _ = self.retag({f"{RC_HOST}/backend:v1.2.0-rc.1": "sha256:rc"}, drift=True) + self.assertEqual(r.returncode, 1) + self.assertIn("not the RC's sha256:rc", r.stdout) + + def test_image_roots(self): + roots = step("cloud-backend-docker-build-push.yaml", "Image roots") + env = {"PROJECT_ID": "p", "TOKEN": "t", "ON_PREM_RC": ""} + r = self.bash(roots, CLOUD_RC=RC, **env) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + r = self.bash(roots, CLOUD_RC="", **env) + with open(os.path.join(self.d, "env")) as fh: + self.assertEqual( + fh.read().split("\n")[:4], + [ + f"IMAGE_ROOT_CLOUD={RC_HOST}", + f"IMAGE_ROOT_ON_PREM={self.STABLE}-on-prem", + f"IMAGE_ROOT_CLOUD={self.STABLE}", + f"IMAGE_ROOT_ON_PREM={self.STABLE}-on-prem", + ], + ) + r = self.bash(roots, CLOUD_RC="bogus", **env) + self.assertEqual(r.returncode, 1) + + def test_image_roots_is_the_same_in_every_build_workflow(self): + # on-prem-backend logs in with the credential helper instead of the token (a long compile outlives it) + def roots(f): + return [ + line + for line in step(f, "Image roots").split("\n") + if not re.search(r"docker login|configure-docker|credential helper", line) + ] + + files = sorted( + f + for f in os.listdir(os.path.join(HERE, "..", "workflows")) + # multi-doc-service is not built by build-push, so it has no RC image + if f.endswith("-docker-build-push.yaml") + and f != "cloud-multi-doc-service-docker-build-push.yaml" + ) + + def env( + f, + ): # the step's RC repository env lines, which gate them on a release build + with open(os.path.join(HERE, "..", "workflows", f)) as fh: + text = fh.read() + block = text[ + text.index("- name: Image roots") : text.index( + "run: |", text.index("- name: Image roots") + ) + ] + return [line.strip() for line in block.split("\n") if "_RC:" in line] + + self.assertEqual(len(files), 11) + self.assertEqual(len(env("cloud-backend-docker-build-push.yaml")), 2) + for f in files: + self.assertEqual(roots(f), roots("cloud-backend-docker-build-push.yaml"), f) + self.assertEqual(env(f), env("cloud-backend-docker-build-push.yaml"), f) + with open(os.path.join(HERE, "..", "workflows", f)) as fh: + text = fh.read() + tags = re.findall(r"tags=\$\{\{ env\.(IMAGE_ROOT_\w+) \}\}", text) + self.assertEqual(len(tags), text.count("tags="), f) # no tag outside a root + # a cloud build pushes its cloud image, and some also their on-prem one; an on-prem build only that + want = ( + [["IMAGE_ROOT_CLOUD"], ["IMAGE_ROOT_CLOUD", "IMAGE_ROOT_ON_PREM"]] + if f.startswith("cloud-") + else [["IMAGE_ROOT_ON_PREM"]] + ) + self.assertIn(tags, want, f) + + def test_on_prem_retag_is_the_cloud_retag(self): + # both jobs' steps share a name, so step() runs the cloud one; the on-prem one differs only in its stable root + # and RC repository variable + def body(n): + with open( + os.path.join(HERE, "..", "workflows", "retag-docker-images.yaml") + ) as fh: + text = fh.read() + at = [ + m.start() + for m in re.finditer( + r"- name: Retag \$\{\{ matrix\.service \}\} image", text + ) + ] + self.assertEqual(len(at), 2) + return text[ + at[n] : text.index("\n\n #", at[n]) + ] # the whole step, env included + + on_prem = body(1).replace('/unstract-on-prem"', '/unstract"') + self.assertEqual( + on_prem.replace("RELSTATE_AR_ON_PREM_RC", "RELSTATE_AR_CLOUD_RC"), body(0) + ) + self.assertIn("RELSTATE_AR_ON_PREM_RC", body(1)) + self.assertIn("crane copy", body(0)) + + class PendingPicksWorkflow(Workflow): """Promote's Plan blocks on open conflict PRs (cloud line or pinned OSS line) and held security/critical fixes unless forced. A failed listing blocks; forcing passes a failed conflict listing (recorded), never a failed diff --git a/.github/release/test_relstate.py b/.github/release/test_relstate.py index 40a0b3f480..abdbc60a6c 100644 --- a/.github/release/test_relstate.py +++ b/.github/release/test_relstate.py @@ -1219,6 +1219,10 @@ def setUp(self): patcher = unittest.mock.patch.dict(os.environ, env) patcher.start() self.addCleanup(patcher.stop) + for k in ("RELSTATE_AR_CLOUD_RC", "RELSTATE_AR_ON_PREM_RC"): + os.environ.pop( + k, None + ) # a test sets _RC itself; the shell's must not leak in (restored on stop) self.urls, self.pages = [], {} urlopen = unittest.mock.patch.object(rs.urllib.request, "urlopen", self.urlopen) urlopen.start() @@ -1236,14 +1240,14 @@ def urlopen(self, req, timeout): ) return io.BytesIO(json.dumps(page).encode()) - def version(self, package, digest, created, *tags): - name = f"{self.REPO}/packages/{package}/versions/sha256:{digest}" + def version(self, package, digest, created, *tags, repo=REPO): + name = f"{repo}/packages/{package}/versions/sha256:{digest}" return { "name": name, "createTime": created, "updateTime": "2030-01-01T00:00:00Z", # differs from createTime: pushed_at must not read it "relatedTags": [ - {"name": f"{self.REPO}/packages/{package}/tags/{t}", "version": name} + {"name": f"{repo}/packages/{package}/tags/{t}", "version": name} for t in tags ], } @@ -1350,6 +1354,136 @@ def test_unset_location_or_token_raises(self): rs.registry_tags("cloud/backend") self.assertEqual(self.urls, []) + RC = "projects/s/locations/l/repositories/rc" + + def test_42_rc_repository_reads_with_the_stable_one(self): + rc_url = f"{rs.AR_API}/{self.RC}/packages/backend/versions" + self.pages[(self.url("backend"), "")] = { + "versions": [ + self.version("backend", "aa", "2026-10-02T10:00:00Z", "v0.192.0") + ] + } + self.pages[(rc_url, "")] = { + "versions": [ + self.version( + "backend", "aa", "2026-10-01T10:00:00Z", "v0.192.0-rc.1", repo=self.RC + ) + ] + } + self.assertEqual( + set(rs.registry_tags("cloud/backend")), {"v0.192.0"} + ) # _RC unset: one repository + with unittest.mock.patch.dict(os.environ, {"RELSTATE_AR_CLOUD_RC": self.RC}): + got = rs.registry_tags("cloud/backend") + self.assertEqual( + got["v0.192.0-rc.1"]["id"], + f"{self.RC}/packages/backend/versions/sha256:aa", + ) + self.assertEqual( + got["v0.192.0"]["id"], f"{self.REPO}/packages/backend/versions/sha256:aa" + ) + self.assertEqual(rs.ar_repositories("cloud"), [self.REPO, self.RC]) + with unittest.mock.patch.dict(os.environ, {"RELSTATE_AR_CLOUD_RC": self.REPO}): + self.assertEqual( + rs.ar_repositories("cloud"), [self.REPO] + ) # the same repository is read once + + def test_43_a_tag_at_two_digests_raises(self): + self.pages[(self.url("backend"), "")] = { + "versions": [ + self.version("backend", "aa", "2026-10-01T10:00:00Z", "v0.192.0-rc.1") + ] + } + self.pages[(f"{rs.AR_API}/{self.RC}/packages/backend/versions", "")] = { + "versions": [ + self.version( + "backend", "bb", "2026-10-01T10:00:00Z", "v0.192.0-rc.1", repo=self.RC + ) + ] + } + with ( + unittest.mock.patch.dict(os.environ, {"RELSTATE_AR_CLOUD_RC": self.RC}), + self.assertRaisesRegex( + RuntimeError, + "tag v0.192.0-rc.1 is .*sha256:aa \\(pushed .*\\) and .*sha256:bb .*promotes the RC repository", + ), + ): + rs.registry_tags("cloud/backend") + + def test_44_rc_repository_404_is_empty_only_when_it_reads(self): + self.pages[(self.url("backend"), "")] = { + "versions": [ + self.version("backend", "aa", "2026-10-01T10:00:00Z", "v0.191.0") + ] + } + with unittest.mock.patch.dict(os.environ, {"RELSTATE_AR_CLOUD_RC": self.RC}): + with self.assertRaisesRegex( + RuntimeError, f"404 and repository {self.RC} not found" + ): + rs.registry_tags("cloud/backend") # a wrong _RC never reads as "no RCs" + self.pages[(f"{rs.AR_API}/{self.RC}", "")] = {"name": self.RC} + self.assertEqual(set(rs.registry_tags("cloud/backend")), {"v0.191.0"}) + + def test_47_a_package_only_in_the_rc_repository(self): + # a service first built on a release line after the split: stable answers 404 for the package but reads + self.pages[(f"{rs.AR_API}/{self.REPO}", "")] = {"name": self.REPO} + self.pages[(f"{rs.AR_API}/{self.RC}/packages/backend/versions", "")] = { + "versions": [ + self.version( + "backend", "aa", "2026-10-01T10:00:00Z", "v0.192.0-rc.1", repo=self.RC + ) + ] + } + with unittest.mock.patch.dict(os.environ, {"RELSTATE_AR_CLOUD_RC": self.RC}): + self.assertEqual(set(rs.registry_tags("cloud/backend")), {"v0.192.0-rc.1"}) + + def test_45_abandon_keeps_the_last_version_per_repository(self): + stable = f"{self.REPO}/packages/backend/versions/sha256:" + rc = f"{self.RC}/packages/backend/versions/sha256:" + reg = { + "cloud/backend": { + "v1.9.0-rc.1": { + "id": stable + "01", + "digest": "sha256:01", + "pushed_at": "x", + }, # before the split + "v1.5.0": {"id": stable + "02", "digest": "sha256:02", "pushed_at": "x"}, + # rc.1's digest again (a reproducible rebuild): still its own version in the RC repository + "v1.9.0-rc.2": {"id": rc + "01", "digest": "sha256:01", "pushed_at": "x"}, + "v1.9.0-rc.3": {"id": rc + "04", "digest": "sha256:04", "pushed_at": "x"}, + } + } + with tempfile.TemporaryDirectory() as d: + rm.git(d, "init", "-q", "-b", "main") + rm.git( + d, + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "base", + ) + rm.git(d, "branch", "release/1.9") + plan = rs.abandon_plan(d, "release/1.9", lambda a: reg.get(a, {})) + self.assertEqual( + [v["id"] for v in plan["versions"]["cloud/backend"]], + [stable + "01", rc + "01"], + ) # the stable repository keeps v1.5.0; the RC repository keeps its last RC + self.assertEqual(plan["kept"], ["cloud/backend:v1.9.0-rc.3"]) + + def test_46_image_link_names_the_rc_repository(self): + with unittest.mock.patch.dict( + os.environ, {"RELSTATE_AR_CLOUD_RC": self.RC, "RELSTATE_AR_CLOUD": self.REPO} + ): + self.assertIn( + "/artifacts/docker/s/l/rc)", + rm.image_link("cloud", "v1-rc.1"), + ) + class Guards(unittest.TestCase): OUTSIDE = ( From 336f4407e48203b7a39a62f86ceea30e53b3765b Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 13:49:58 +0530 Subject: [PATCH 34/37] Prefix the new release workflows' display names with V2 The new workflows run next to the old ones until the old ones are deleted, so the prefix tells them apart in the Actions list. File names and job names are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/workflows/backport.yaml | 2 +- .github/workflows/lint-pr.yaml | 2 +- .github/workflows/release-branch-ci.yaml | 2 +- .github/workflows/release-branch-only.yaml | 2 +- .github/workflows/release-tooling-tests.yaml | 2 +- .github/workflows/state.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 40c370e1b3..4c33ead09b 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -1,4 +1,4 @@ -name: Backport +name: V2 Backport # Expressions have no substring, so a push run names the full sha. run-name: >- ${{ github.event_name == 'pull_request_target' diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml index 616d478517..598e0fba89 100644 --- a/.github/workflows/lint-pr.yaml +++ b/.github/workflows/lint-pr.yaml @@ -1,4 +1,4 @@ -name: Lint PR +name: V2 Lint PR # Every PR into main or a release branch carries a recognised `[TYPE]` title prefix, ticket optional # (`ABC-123 [FIX] ...` or `[FIX] ...`), unless a fix/ label gives its class: at most one of fix/security, diff --git a/.github/workflows/release-branch-ci.yaml b/.github/workflows/release-branch-ci.yaml index c09dc04c07..c253d16797 100644 --- a/.github/workflows/release-branch-ci.yaml +++ b/.github/workflows/release-branch-ci.yaml @@ -1,4 +1,4 @@ -name: Release branch CI +name: V2 Release branch CI # Every commit after the cut must be a pick of a main commit (or its decline), release-branch-only, or # build-generated. Anything else fails. diff --git a/.github/workflows/release-branch-only.yaml b/.github/workflows/release-branch-only.yaml index 97c69b834c..c205757362 100644 --- a/.github/workflows/release-branch-only.yaml +++ b/.github/workflows/release-branch-only.yaml @@ -1,4 +1,4 @@ -name: Release branch only +name: V2 Release branch only # On the `release-branch-only` label on a PR into release/**: one empty bot commit carrying # `Release-branch-only: ` (ticket from the title, else `#`) goes onto the PR head, unless a commit diff --git a/.github/workflows/release-tooling-tests.yaml b/.github/workflows/release-tooling-tests.yaml index e0c20dd214..493ba5ee89 100644 --- a/.github/workflows/release-tooling-tests.yaml +++ b/.github/workflows/release-tooling-tests.yaml @@ -1,4 +1,4 @@ -name: Release tooling tests +name: V2 Release tooling tests # Runs every .github/release test suite when the release tooling or a workflow that runs it changes. # The suites use only temporary local git repos and mocks: no token, no network. diff --git a/.github/workflows/state.yaml b/.github/workflows/state.yaml index 78057d372c..ff7ba89719 100644 --- a/.github/workflows/state.yaml +++ b/.github/workflows/state.yaml @@ -1,4 +1,4 @@ -name: Release state +name: V2 Release state # Read-only: prints every release/X.Y branch with its state. It writes nothing. on: From 44f540171b1daf282fa45c6b1f6633aa7a6f48ec Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 14:08:15 +0530 Subject: [PATCH 35/37] Pass LD_LIBRARY_PATH to the workflow-step tests GuardSteps runs workflow steps with a clean environment. On the runner, the setup-python interpreter needs LD_LIBRARY_PATH to load libpython, so every step that calls python3 failed there. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/test_oss.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index c5c0894ea4..9ad9f3f327 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -1193,6 +1193,8 @@ def bash(self, script, **env): ["bash", "-eo", "pipefail", "-c", script], env={ "PATH": f"{self.d}:{os.environ['PATH']}", + # setup-python's interpreter loads libpython from here + **{k: v for k, v in os.environ.items() if k == "LD_LIBRARY_PATH"}, "GITHUB_OUTPUT": os.path.join(self.d, "out"), "GITHUB_STEP_SUMMARY": os.path.join(self.d, "summary"), "GITHUB_ENV": os.path.join(self.d, "env"), From fecbff88428b2238b14680e6b21f66ba73bd8be6 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 19:43:14 +0530 Subject: [PATCH 36/37] Test the build workflows' skip of an already pushed RC Mirrors the release tooling from the cloud repo: test_oss gains tests for the step that skips a release-line RC build whose images are already pushed. They run only where the build workflows exist. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/test_oss.py | 112 +++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 9ad9f3f327..1779564288 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -1150,7 +1150,7 @@ class GuardSteps(unittest.TestCase): update step, retag's copy and the build workflows' image roots. Stubs: curl (the GET answers get.json with the HTTP code in code, the PATCH body lands in patch.json); crane (img/ holds an image's digest, a missing one prints crane_err and fails, each copy is logged to copies, drift makes a copy land at another digest); docker - (login only). + (login; buildx imagetools inspect and create over the same img/ and copies, inspect_err failing every inspect). """ APP = { @@ -1184,7 +1184,18 @@ def setUp(self): esac """) with open(os.path.join(d, "docker"), "w") as fh: - fh.write("#!/bin/sh\ncat > /dev/null\n") + fh.write(f"""#!/bin/sh +img() {{ echo "{d}/img/$(echo "$1" | tr '/:' '__')"; }} +case "$1 $2 $3" in + "buildx imagetools inspect") + [ ! -e {d}/inspect_err ] || {{ cat {d}/inspect_err; exit 1; }} + [ -e "$(img "$4")" ] || {{ echo "ERROR: $4: not found"; exit 1; }} + cat "$(img "$4")";; + "buildx imagetools create") echo "$6 $5" >> {d}/copies; cp "$(img "${{6%@*}}")" "$(img "$5")" + [ ! -e {d}/drift ] || echo sha256:other > "$(img "$5")";; + *) cat > /dev/null;; +esac +""") for tool in ("crane", "docker"): os.chmod(os.path.join(d, tool), 0o755) @@ -1457,6 +1468,103 @@ def test_image_roots(self): r = self.bash(roots, CLOUD_RC="bogus", **env) self.assertEqual(r.returncode, 1) + def skip(self, images, tag="v1.2.0-rc.1", err="", drift=False): + """Run the runner's Skip if already pushed step over images (ref -> digest).""" + for image, digest in images.items(): + with open( + os.path.join(self.d, "img", image.replace("/", "_").replace(":", "_")), + "w", + ) as fh: + fh.write(digest) + if err: + with open(os.path.join(self.d, "inspect_err"), "w") as fh: + fh.write(err) + if drift: + open(os.path.join(self.d, "drift"), "w").close() + r = self.bash( + step("cloud-runner-docker-build-push.yaml", "Skip if already pushed"), + TAG=tag, + IMAGES="c/runner o/runner", + ) + out = {} + for name in ("out", "copies"): + path = os.path.join(self.d, name) + with open(path) if os.path.exists(path) else io.StringIO() as fh: + out[name] = fh.read() + return r, out["out"], out["copies"] + + def test_skip_when_every_root_has_the_rc(self): + r, out, copies = self.skip( + {"c/runner:v1.2.0-rc.1": "sha256:a", "o/runner:v1.2.0-rc.1": "sha256:a"} + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual((out, copies), ("skip=true\n", "")) + + def test_build_when_no_root_has_the_rc(self): + r, out, copies = self.skip({}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual((out, copies), ("", "")) + self.assertIn("not pushed yet", r.stdout) + + def test_copy_the_rc_to_a_root_that_lacks_it(self): + r, out, copies = self.skip({"o/runner:v1.2.0-rc.1": "sha256:a"}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual(out, "skip=true\n") + self.assertEqual(copies, "o/runner:v1.2.0-rc.1@sha256:a c/runner:v1.2.0-rc.1\n") + with open(os.path.join(self.d, "img", "c_runner_v1.2.0-rc.1")) as fh: + self.assertEqual(fh.read().strip(), "sha256:a") + + def test_copy_refuses_another_digest(self): + r, out, _ = self.skip({"c/runner:v1.2.0-rc.1": "sha256:a"}, drift=True) + self.assertEqual(r.returncode, 1) + self.assertEqual(out, "") + self.assertIn("landed at sha256:other, not sha256:a", r.stdout) + + def test_skip_fails_when_a_root_cannot_be_read(self): + r, out, _ = self.skip({}, err="ERROR: unauthorized") + self.assertEqual(r.returncode, 1) + self.assertEqual(out, "") + self.assertIn( + "cannot inspect c/runner:v1.2.0-rc.1: ERROR: unauthorized", r.stdout + ) + + def test_skip_leaves_other_tags_alone(self): + for tag in ("rc.12", "v1.2.0", "v1.2.0-rc.1-snapshot", "dev-abc"): + r, out, _ = self.skip( + {f"c/runner:{tag}": "sha256:a", f"o/runner:{tag}": "sha256:a"}, tag=tag + ) + self.assertEqual((r.returncode, out), (0, ""), tag) + + def test_skip_if_already_pushed_is_the_same_in_every_build_workflow(self): + # the same script before every bake, gated on a release build, its images the ones the bake tags + files = sorted( + f + for f in os.listdir(os.path.join(HERE, "..", "workflows")) + if f.endswith("-docker-build-push.yaml") + and f != "cloud-multi-doc-service-docker-build-push.yaml" + ) + self.assertEqual(len(files), 11) + want = step("cloud-backend-docker-build-push.yaml", "Skip if already pushed") + for f in files: + self.assertEqual(step(f, "Skip if already pushed"), want, f) + with open(os.path.join(HERE, "..", "workflows", f)) as fh: + text = fh.read() + at = text.index("- name: Skip if already pushed") + block = text[at : text.index("run: |", at)] + self.assertIn("id: pushed\n", block, f) + self.assertIn("if: inputs.event == 'release'\n", block, f) + images = re.search(r"IMAGES: (.*)", block).group(1) + tags = re.findall(r"tags=(\$\{\{ env\.IMAGE_ROOT_\w+ \}\}/[\w-]+):", text) + self.assertEqual(images, " ".join(tags), f) + bake = text.index("uses: docker/bake-action@v7") + self.assertLess(at, bake, f) + self.assertEqual(text.count("uses: docker/bake-action@v7"), 1, f) + self.assertEqual( + text[:bake].rstrip().split("\n")[-1].strip(), + "if: steps.pushed.outputs.skip != 'true'", + f, + ) + def test_image_roots_is_the_same_in_every_build_workflow(self): # on-prem-backend logs in with the credential helper instead of the token (a long compile outlives it) def roots(f): From cbb4ed45a8971942a632dbd4cae280cbefd9510d Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 20:02:36 +0530 Subject: [PATCH 37/37] Test the revision check of the build workflows' skip step Mirrors the release tooling from the cloud repo: test_oss gains tests for the skip step refusing an RC image built from another commit. They run only where the build workflows exist. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XbT4FqysueCepjZ1uCpPk1 --- .github/release/test_oss.py | 106 +++++++++++++++++++++++++++++++----- 1 file changed, 91 insertions(+), 15 deletions(-) diff --git a/.github/release/test_oss.py b/.github/release/test_oss.py index 1779564288..7ecd5298e6 100644 --- a/.github/release/test_oss.py +++ b/.github/release/test_oss.py @@ -1150,7 +1150,8 @@ class GuardSteps(unittest.TestCase): update step, retag's copy and the build workflows' image roots. Stubs: curl (the GET answers get.json with the HTTP code in code, the PATCH body lands in patch.json); crane (img/ holds an image's digest, a missing one prints crane_err and fails, each copy is logged to copies, drift makes a copy land at another digest); docker - (login; buildx imagetools inspect and create over the same img/ and copies, inspect_err failing every inspect). + (login; buildx imagetools inspect and create over the same img/ and copies, inspect_err failing every inspect, + img/.json the image config that --format '{{json .Image}}' prints). """ APP = { @@ -1189,9 +1190,10 @@ def setUp(self): case "$1 $2 $3" in "buildx imagetools inspect") [ ! -e {d}/inspect_err ] || {{ cat {d}/inspect_err; exit 1; }} - [ -e "$(img "$4")" ] || {{ echo "ERROR: $4: not found"; exit 1; }} - cat "$(img "$4")";; + [ -e "$(img "${{4%@*}}")" ] || {{ echo "ERROR: $4: not found"; exit 1; }} + if [ "$6" = '{{{{json .Image}}}}' ]; then cat "$(img "${{4%@*}}").json"; else cat "$(img "$4")"; fi;; "buildx imagetools create") echo "$6 $5" >> {d}/copies; cp "$(img "${{6%@*}}")" "$(img "$5")" + cp "$(img "${{6%@*}}").json" "$(img "$5").json" [ ! -e {d}/drift ] || echo sha256:other > "$(img "$5")";; *) cat > /dev/null;; esac @@ -1468,14 +1470,22 @@ def test_image_roots(self): r = self.bash(roots, CLOUD_RC="bogus", **env) self.assertEqual(r.returncode, 1) - def skip(self, images, tag="v1.2.0-rc.1", err="", drift=False): - """Run the runner's Skip if already pushed step over images (ref -> digest).""" + LABELLED = '{"config": {"Labels": {"org.opencontainers.image.revision": "r1"}}, "rootfs": {}}' + + def skip(self, images, tag="v1.2.0-rc.1", err="", drift=False, config=LABELLED): + """Run the runner's Skip at commit r1 over images (ref -> digest, or ref -> (digest, config)).""" for image, digest in images.items(): - with open( - os.path.join(self.d, "img", image.replace("/", "_").replace(":", "_")), - "w", - ) as fh: + digest, cfg = digest if isinstance(digest, tuple) else (digest, config) + path = os.path.join(self.d, "img", image.replace("/", "_").replace(":", "_")) + with open(path, "w") as fh: fh.write(digest) + with open(path + ".json", "w") as fh: + fh.write(cfg) + git = os.path.join(self.d, "git") + os.makedirs(git, exist_ok=True) + with open(os.path.join(git, "git"), "w") as fh: + fh.write('#!/bin/sh\n[ "$*" = "rev-parse HEAD" ] && echo r1\n') + os.chmod(os.path.join(git, "git"), 0o755) if err: with open(os.path.join(self.d, "inspect_err"), "w") as fh: fh.write(err) @@ -1485,6 +1495,7 @@ def skip(self, images, tag="v1.2.0-rc.1", err="", drift=False): step("cloud-runner-docker-build-push.yaml", "Skip if already pushed"), TAG=tag, IMAGES="c/runner o/runner", + PATH=f"{git}:{self.d}:{os.environ['PATH']}", ) out = {} for name in ("out", "copies"): @@ -1498,18 +1509,18 @@ def test_skip_when_every_root_has_the_rc(self): {"c/runner:v1.2.0-rc.1": "sha256:a", "o/runner:v1.2.0-rc.1": "sha256:a"} ) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) - self.assertEqual((out, copies), ("skip=true\n", "")) + self.assertEqual((out, copies), ("revision=r1\nskip=true\n", "")) def test_build_when_no_root_has_the_rc(self): r, out, copies = self.skip({}) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) - self.assertEqual((out, copies), ("", "")) + self.assertEqual((out, copies), ("revision=r1\n", "")) self.assertIn("not pushed yet", r.stdout) def test_copy_the_rc_to_a_root_that_lacks_it(self): r, out, copies = self.skip({"o/runner:v1.2.0-rc.1": "sha256:a"}) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) - self.assertEqual(out, "skip=true\n") + self.assertEqual(out, "revision=r1\nskip=true\n") self.assertEqual(copies, "o/runner:v1.2.0-rc.1@sha256:a c/runner:v1.2.0-rc.1\n") with open(os.path.join(self.d, "img", "c_runner_v1.2.0-rc.1")) as fh: self.assertEqual(fh.read().strip(), "sha256:a") @@ -1517,23 +1528,74 @@ def test_copy_the_rc_to_a_root_that_lacks_it(self): def test_copy_refuses_another_digest(self): r, out, _ = self.skip({"c/runner:v1.2.0-rc.1": "sha256:a"}, drift=True) self.assertEqual(r.returncode, 1) - self.assertEqual(out, "") + self.assertEqual(out, "revision=r1\n") self.assertIn("landed at sha256:other, not sha256:a", r.stdout) def test_skip_fails_when_a_root_cannot_be_read(self): r, out, _ = self.skip({}, err="ERROR: unauthorized") self.assertEqual(r.returncode, 1) - self.assertEqual(out, "") + self.assertEqual(out, "revision=r1\n") self.assertIn( "cannot inspect c/runner:v1.2.0-rc.1: ERROR: unauthorized", r.stdout ) def test_skip_leaves_other_tags_alone(self): for tag in ("rc.12", "v1.2.0", "v1.2.0-rc.1-snapshot", "dev-abc"): + self.setUp() r, out, _ = self.skip( {f"c/runner:{tag}": "sha256:a", f"o/runner:{tag}": "sha256:a"}, tag=tag ) - self.assertEqual((r.returncode, out), (0, ""), tag) + self.assertEqual((r.returncode, out), (0, "revision=r1\n"), tag) + + def test_skip_refuses_an_image_of_another_commit(self): + other = self.LABELLED.replace("r1", "r0") + for images in ( + {"c/runner:v1.2.0-rc.1": ("sha256:a", other)}, + { + "c/runner:v1.2.0-rc.1": "sha256:a", + "o/runner:v1.2.0-rc.1": ("sha256:a", other), + }, + ): + with self.subTest(images=images): + self.setUp() + r, out, copies = self.skip(images) + self.assertEqual(r.returncode, 1) + self.assertEqual((out, copies), ("revision=r1\n", "")) + self.assertIn( + "was built from r0, not r1: the tag was used before", r.stdout + ) + + def test_skip_refuses_an_unlabelled_image(self): + r, out, copies = self.skip( + {"c/runner:v1.2.0-rc.1": "sha256:a"}, config='{"config": {}, "rootfs": {}}' + ) + self.assertEqual(r.returncode, 1) + self.assertEqual((out, copies), ("revision=r1\n", "")) + self.assertIn("was built from an unlabelled commit, not r1", r.stdout) + + def test_skip_reads_the_label_of_every_platform(self): + def per_platform(*revs): + return json.dumps( + { + f"linux/p{n}": { + "config": {"Labels": {"org.opencontainers.image.revision": rev}} + } + for n, rev in enumerate(revs) + } + ) + + r, out, _ = self.skip( + {"c/runner:v1.2.0-rc.1": "sha256:a"}, config=per_platform("r1", "r1") + ) + self.assertEqual( + (r.returncode, out), (0, "revision=r1\nskip=true\n"), r.stdout + r.stderr + ) + self.setUp() + r, out, _ = self.skip( + {"c/runner:v1.2.0-rc.1": "sha256:a"}, config=per_platform("r1", "r0") + ) + self.assertEqual(r.returncode, 1) + self.assertIn("was built from r0 r1, not r1", r.stdout) def test_skip_if_already_pushed_is_the_same_in_every_build_workflow(self): # the same script before every bake, gated on a release build, its images the ones the bake tags @@ -1556,6 +1618,20 @@ def test_skip_if_already_pushed_is_the_same_in_every_build_workflow(self): images = re.search(r"IMAGES: (.*)", block).group(1) tags = re.findall(r"tags=(\$\{\{ env\.IMAGE_ROOT_\w+ \}\}/[\w-]+):", text) self.assertEqual(images, " ".join(tags), f) + # the bake labels its images with the commit the step compares + self.assertEqual( + re.findall( + r"\n +(\*|\w+)\.labels\.org\.opencontainers\.image\.revision=(.*)\n", + text, + ), + [ + ( + re.search(r"\n +(\*|\w+)\.tags=", text).group(1), + "${{ steps.pushed.outputs.revision || github.sha }}", + ) + ], + f, + ) bake = text.index("uses: docker/bake-action@v7") self.assertLess(at, bake, f) self.assertEqual(text.count("uses: docker/bake-action@v7"), 1, f)