From 51d9c04fa57ed34ede910599185a7b44fca4396b Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:25:58 +0300 Subject: [PATCH 1/3] refactor(pruner): collect units with one find and one awk per table rule collect_units forked a find|awk pair plus stat subshells per evictable unit, tens of thousands of times on Gradle/bun caches. It now runs one find -printf per dir rule and one awk that sums size and newest file atime per unit (units sit at the glob's depth and find lists a directory before its contents), taking bun's package.json atime from the same stream. STORE_TABLE gets an opts column (shared, key=) replacing the hard-coded actions exclusion and the bun unit_key override; the remaining per-store behaviour (rustup default protection, toolcache .complete markers, bun symlinks) sits in one store_hook dispatcher. Adds tests/bench/pruner-100k.sh, a timing of one pass over ~100k synthetic units. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/build.yml | 2 +- pruner.sh | 236 +++++++++++++++++++++++------------- tests/bench/pruner-100k.sh | 60 +++++++++ 3 files changed, 214 insertions(+), 84 deletions(-) create mode 100755 tests/bench/pruner-100k.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 783e42b..6d65f1a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: # nullglob: a pattern that matches nothing (e.g. hosts/*.sh) must # drop out instead of being passed to shellcheck as a literal. shopt -s nullglob - files=(*.sh hosts/*.sh tests/*.sh) + files=(*.sh hosts/*.sh tests/*.sh tests/bench/*.sh) if [ "${#files[@]}" -eq 0 ]; then echo "::error::No shell scripts found to check" exit 1 diff --git a/pruner.sh b/pruner.sh index 17f0681..28c0f18 100755 --- a/pruner.sh +++ b/pruner.sh @@ -66,7 +66,7 @@ DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } -# Evictable units per store, one rule per line: store|kind|glob|skip +# Evictable units per store, one rule per line: store|kind|glob|skip|opts # kind d: every directory matching glob (relative to the store dir, never a # symlink) is one unit, removed whole; skip is a case pattern for the # relative path of units to leave alone. Glob "." makes the whole @@ -74,40 +74,44 @@ log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } # emptied, but its subdirs are kept, as container symlinks point there. # kind f: every regular file named skip (a find -name pattern) under the dir # glob is one unit. +# opts: comma-separated. shared: not a runner-slot store (the action archive +# cache). key=: LRU order of a dir unit is the atime of / +# (the unit's mtime when it has none), not the newest atime of any file. +# Globs use * and ? only; like the shell, they skip names starting with a dot. # Anything no rule matches (bun's bin/ and @t@ piles, cargo's bin/ and registry -# index, rustup's settings.toml, ...) is never touched. Hooks for the few stores -# needing more than a rule are in unit_key / unit_protected / unit_pre_evict / -# unit_post_evict. Leftovers of killed bun extractions (*.tmp, *.part) are -# pre-job.sh's job: it sweeps them before the next job of a crashed slot. +# index, rustup's settings.toml, ...) is never touched. The few stores needing +# more than a rule are in store_hook. Leftovers of killed bun extractions +# (*.tmp, *.part) are pre-job.sh's job: it sweeps them before the next job of a +# crashed slot. STORE_TABLE=' -bun|d|install/cache/*@*|install/cache/@* -bun|d|install/cache/@*/*@*|install/cache/@t@/* -npm|f|_cacache|* -pnpm|d|.| -gradle|d|caches/modules-2/files-2.1/*/*/*| -gradle|d|caches/transforms-*/*| -gradle|d|wrapper/dists/*| -gradle|d|caches/jars-*/*| -gradle|f|caches/build-cache-1|* -cargo|f|registry/cache|*.crate -cargo|d|registry/src/*/*| -cargo|d|git/checkouts/*| -cargo|d|git/db/*| -rustup|d|toolchains/*| -konan|d|dependencies/*| -konan|d|kotlin-native-prebuilt-*| -mise-data|d|installs/*/*| -pub|d|hosted/*/*| -pub|d|git/*| -toolcache|d|*/*| -mise-cache|f|.|* -hk|f|.|* -actions|f|.|* +bun|d|install/cache/*@*|install/cache/@*|key=package.json +bun|d|install/cache/@*/*@*|install/cache/@t@/*|key=package.json +npm|f|_cacache|*| +pnpm|d|.|| +gradle|d|caches/modules-2/files-2.1/*/*/*|| +gradle|d|caches/transforms-*/*|| +gradle|d|wrapper/dists/*|| +gradle|d|caches/jars-*/*|| +gradle|f|caches/build-cache-1|*| +cargo|f|registry/cache|*.crate| +cargo|d|registry/src/*/*|| +cargo|d|git/checkouts/*|| +cargo|d|git/db/*|| +rustup|d|toolchains/*|| +konan|d|dependencies/*|| +konan|d|kotlin-native-prebuilt-*|| +mise-data|d|installs/*/*|| +pub|d|hosted/*/*|| +pub|d|git/*|| +toolcache|d|*/*|| +mise-cache|f|.|*| +hk|f|.|*| +actions|f|.|*|shared ' # Stores of a runner slot (everything but the shared action archive cache). slot_stores() { - awk -F'|' 'NF > 3 && $1 != "actions" && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" + awk -F'|' 'NF > 3 && $5 !~ /(^|,)shared(,|$)/ && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" } dir_bytes() { @@ -119,57 +123,132 @@ rustup_default() { sed -n 's/^default_toolchain[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1/settings.toml" 2>/dev/null | head -n 1 } +# The per-store behaviour a table rule cannot express: store_hook . +# protect prints the names of units never to evict, one +# per line (a unit named N or N-* is kept) +# pre before a dir unit is removed +# post after it is gone +store_hook() { + local base + case "$1:$2" in + # `stable` is the dir stable-, hence the N-* match. + protect:rustup) rustup_default "$3" ;; + # /.complete says / is whole. Drop the markers first + # so an interrupted removal never leaves a "complete" half tree. + pre:toolcache) find "$3" -mindepth 1 -maxdepth 1 -name '*.complete' -delete 2>/dev/null ;; + # Drop the /@@@ symlink pointing at the unit, and the dir if empty. + post:bun) + base="${3##*/}" + rm -f "${3%/*}/${base%%@*}/${base#*@}" 2>/dev/null + rmdir "${3%/*}/${base%%@*}" 2>/dev/null + ;; + esac + return 0 +} + # bun treats install/cache/@@@@/ as complete whenever the folder # exists, so a file evicted from it is silently missing from every later # node_modules (next@16.3.8 lost dist/compiled/server-only/empty.js, breaking # every Next build). 0-byte files went first: copying them never reads, so # their atime never moved. Hence whole packages, least recently used first by -# the atime of their package.json (read on every install and import). +# the atime of their package.json (read on every install and import): the +# key=package.json option of the bun rules. -# LRU key override for dir unit $2 of store $1; prints nothing to keep the -# newest-atime-of-any-file key. -unit_key() { - [ "$1" = bun ] || return 0 - stat -c %X "$2/package.json" 2>/dev/null || stat -c %Y "$2" 2>/dev/null +# awk program of collect_units. Input: find lines "%P\t%y\t%A@\t%T@\t%s\t%d" +# (path below the rule's prefix, type, atime, mtime, size, depth). Variables: +# store, sd (store dir), pre (rule prefix), glob (rule glob below pre), depth +# (its component count), skip (case pattern on the path below sd), keyfile (see +# key= in STORE_TABLE), protect (names never evicted, one per line). +# mawk compatible (this is Ubuntu's default awk): no gensub, no NUL in strings. +# shellcheck disable=SC2016 # awk program, not shell +UNITS_AWK=' +# Shell glob (* and ? only; a leading dot is never matched) as an anchored regex. +function glob_re(g, i, ch, out, start) { + out = "^"; start = 1 + for (i = 1; i <= length(g); i++) { + ch = substr(g, i, 1) + if (ch == "*") out = out (start ? "([^./][^/]*)?" : "[^/]*") + else if (ch == "?") out = out (start ? "[^./]" : "[^/]") + else if (index(".+(){}|^$[]\\", ch)) out = out "\\" ch + else out = out ch + start = (ch == "/") + } + return out "$" } - -# Succeeds when unit $3 (relative to dir $2 of store $1) must not be evicted. -unit_protected() { - local def - [ "$1" = rustup ] || return 1 - def=$(rustup_default "$2") - [ -n "$def" ] || return 1 - case "${3##*/}" in - "$def" | "$def"-*) return 0 ;; # `stable` is the dir stable- - esac - return 1 +# `case` pattern (* matches / too) as an anchored regex. +function case_re(g, i, ch, out) { + out = "^" + for (i = 1; i <= length(g); i++) { + ch = substr(g, i, 1) + if (ch == "*") out = out ".*" + else if (ch == "?") out = out "." + else if (index(".+(){}|^$[]\\", ch)) out = out "\\" ch + else out = out ch + } + return out "$" } - -# toolcache: /.complete says / is whole. Drop the markers -# first so an interrupted removal never leaves a "complete" half tree. -unit_pre_evict() { - [ "$1" = toolcache ] || return 0 - find "$2" -mindepth 1 -maxdepth 1 -name '*.complete' -delete 2>/dev/null +function flush( key) { + if (cur == "") return + key = keyfile == "" ? maxf : (keyat != "" ? keyat : mtime) + printf "%.6f\t%.0f\t%s\t%s\t%s%c", key, size, store, stop, sd "/" rel, 0 + cur = "" } - -# After unit $2 of store $1 is gone. -unit_post_evict() { - local base - [ "$1" = bun ] || return 0 - # Drop the /@@@ symlink pointing at it, and the dir if empty. - base="${2##*/}" - rm -f "${2%/*}/${base%%@*}/${base#*@}" 2>/dev/null - rmdir "${2%/*}/${base%%@*}" 2>/dev/null +function protected(rel, base, i) { + base = rel; sub(/^.*\//, "", base) + for (i = 1; i <= nprot; i++) + if (prot[i] != "" && (base == prot[i] || index(base, prot[i] "-") == 1)) return 1 return 0 } +BEGIN { + FS = "\t" + root = (glob == ".") + if (root) depth = 0 + gre = glob_re(glob) + sre = skip == "" ? "" : case_re(skip) + nprot = split(protect, prot, "\n") + stop = sd (pre == "" ? "" : "/" pre) +} +NF != 6 { next } +{ + d = $6 + 0 + if (d > depth) { # below the unit level: part of the current unit, if any + if (cur == "") next + if (!root && substr($1, 1, plen) != cp) next + size += $5 + if ($2 == "f") { + if ($3 + 0 > maxf) maxf = $3 + 0 + if (keyfile != "" && $1 == cp keyfile) keyat = $3 + } + next + } + flush() + if (d < depth || $2 != "d") next + if (!root && $1 !~ gre) next + rel = root ? "." : (pre == "" ? $1 : pre "/" $1) + if (sre != "" && rel ~ sre) next + if (protected(rel)) next + cur = rel; cp = $1 "/"; plen = length(cp) + if (root) cp = "" + size = $5; maxf = 0; keyat = ""; mtime = $4 +} +END { flush() } +' # Print NUL-terminated "keysizestorestoppath" records, one # per evictable unit of store $1 living in dir $2. key is the LRU order (atime, # seconds); stop is the dir above which emptied parents are not removed. +# +# One find per dir rule lists everything below the rule's fixed prefix, and one +# awk turns that listing into units. A unit sits at a fixed depth (the glob's +# component count) and find lists a directory right before its contents, so a +# unit's size, newest file atime and package.json atime are summed up as the +# lines go by: no per-unit process. collect_units() { - local store="$1" sd="$2" s kind glob skip pre stop + local store="$1" sd="$2" s kind glob skip opts pre rest key protect + local -a parts [ -d "$sd" ] || return 0 - while IFS='|' read -r s kind glob skip; do + protect=$(store_hook protect "$store" "$sd") + while IFS='|' read -r s kind glob skip opts; do [ "$s" = "$store" ] || continue if [ "$kind" = f ]; then [ -d "$sd/$glob" ] || continue @@ -181,24 +260,15 @@ collect_units() { */*) pre="${pre%/*}" ;; *) pre="" ;; esac - stop="$sd${pre:+/$pre}" - ( - cd "$sd" 2>/dev/null || exit 0 - shopt -s nullglob - # shellcheck disable=SC2086 # glob expansion is the point - for rel in $glob; do - if [ ! -d "$rel" ] || [ -L "$rel" ]; then continue; fi - # shellcheck disable=SC2254 # skip is a pattern - case "$rel" in $skip) continue ;; esac - if unit_protected "$store" "$sd" "$rel"; then continue; fi - stats=$(find "$rel" -printf '%y\t%A@\t%s\n' 2>/dev/null | - awk -F'\t' '{ s += $3; if ($1 == "f" && $2 + 0 > m) m = $2 + 0 } END { printf "%.6f\t%.0f\n", m, s }') - key="${stats%%$'\t'*}" - t=$(unit_key "$store" "$rel") - [ -z "$t" ] || key="$t" - printf '%s\t%s\t%s\t%s\t%s\0' "$key" "${stats#*$'\t'}" "$store" "$stop" "$sd/$rel" - done - ) + rest="${glob#"$pre"}" + rest="${rest#/}" + IFS=/ read -ra parts <<<"$rest" + key="" + case ",$opts," in *,key=*) key="${opts#*key=}"; key="${key%%,*}" ;; esac + [ -d "$sd${pre:+/$pre}" ] || continue + find "$sd${pre:+/$pre}" -printf '%P\t%y\t%A@\t%T@\t%s\t%d\n' 2>/dev/null | + awk -F'\t' -v store="$store" -v sd="$sd" -v pre="$pre" -v glob="$rest" -v depth="${#parts[@]}" \ + -v skip="$skip" -v keyfile="$key" -v protect="$protect" "$UNITS_AWK" done <<<"$STORE_TABLE" } @@ -210,9 +280,9 @@ evict_unit() { find "${path%/.}" -mindepth 2 -delete 2>/dev/null find "${path%/.}" -mindepth 1 -maxdepth 1 ! -type d -delete 2>/dev/null elif [ -d "$path" ] && [ ! -L "$path" ]; then - unit_pre_evict "$store" "$path" + store_hook pre "$store" "$path" rm -rf "$path" 2>/dev/null - unit_post_evict "$store" "$path" + store_hook post "$store" "$path" p="${path%/*}" while [ "$p" != "$stop" ] && [ "${p#"$stop"/}" != "$p" ]; do rmdir "$p" 2>/dev/null || break diff --git a/tests/bench/pruner-100k.sh b/tests/bench/pruner-100k.sh new file mode 100755 index 0000000..37db701 --- /dev/null +++ b/tests/bench/pruner-100k.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Timing of one pruner pass over a synthetic cache of ~100k evictable units +# (50k bun packages, 30k gradle module versions, 20k npm cache files), with a +# budget at 90% of its size so about a tenth gets evicted. Not a test: CI only +# runs tests/*.sh. Needs GNU tools; on macOS: +# docker run --rm -v "$PWD:/w" -w /w ubuntu:24.04 bash tests/bench/pruner-100k.sh [scale-percent] +# Prints the unit count, the time of the collect phase alone, and of a full +# prune_caches pass (collect + sort + evict). +set -uo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +scale="${1:-100}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# shellcheck source=pruner.sh +source "${PRUNER_SH:-$here/pruner.sh}" +RUNNER_DATA_DIR="$work/data" +ACTION_CACHE_DIR="$work/no-action-cache" +CACHE_BUDGET_PCT=100 + +# mkfiles : read dirs on stdin, create each with a file in it, +# atimes spread over 50 ages. +mkfiles() { + local file="$1" list="$work/list" k + cat >"$list" + xargs -r -a "$list" -n 2000 -P 4 mkdir -p + for k in $(seq 0 49); do + awk -v k="$k" -v f="$file" 'NR % 50 == k {print $0 "/" f}' "$list" >"$work/files" + # shellcheck disable=SC2016 # expands in the child shell + xargs -r -a "$work/files" -n 2000 sh -c 'for p; do printf "%1024s" x >"$p"; done' _ + xargs -r -a "$work/files" -n 2000 touch -a -d "$((k + 1)) days ago" + done +} + +slot="$RUNNER_DATA_DIR/slots/1" +cache="$slot/cache" +mkdir -p "$cache" "$slot/work" + +seq 1 $((50000 * scale / 100)) | + awk -v d="$cache/bun/install/cache" '{print d "/pkg" $1 "@1.0.0@@@1"}' | mkfiles package.json +seq 1 $((30000 * scale / 100)) | + awk -v d="$cache/gradle/caches/modules-2/files-2.1" '{print d "/org.g" $1 "/lib/1.0.0/h"}' | mkfiles lib.jar +seq 1 $((20000 * scale / 100)) | + awk -v d="$cache/npm/_cacache" '{print d "/c" $1}' | mkfiles f +echo "fixture: $(find "$cache" -type f | wc -l) files" + +CACHE_BUDGET_BYTES=$(($(dir_bytes "$cache") * 9 / 10)) + +start=$EPOCHREALTIME +units=$(for s in bun gradle npm; do collect_units "$s" "$cache/$s"; done | tr -cd '\0' | wc -c) +end=$EPOCHREALTIME +echo "units: $units" +echo "collect: $(awk -v a="$start" -v b="$end" 'BEGIN { printf "%.1f", b - a }') s" + +start=$EPOCHREALTIME +prune_caches >"$work/prune.log" 2>&1; rc=$? +end=$EPOCHREALTIME +echo "pass: $(awk -v a="$start" -v b="$end" 'BEGIN { printf "%.1f", b - a }') s (collect + sort + evict, rc=$rc)" +tail -n 3 "$work/prune.log" From 809efab1cd9f8a83e8ad629a2e1fcd8e402c57a3 Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:26:02 +0300 Subject: [PATCH 2/3] refactor(ci): read the Dockerfile once and generate image tags in the matrix jq The matrix job extracts the Dockerfile versions once and exposes runner_tag as an output; the build and merge jobs of deploy.yml and the build job of build.yml no longer repeat the extraction. deploy.yml's matrix entries now carry their own tag list (merge) and the tag baked into the image (build), replacing the per-line enable=${{ matrix.variant == ... }} and the image-identity metadata step. The variant/target/platform/cache-scope table moves to .github/jq/matrix.jq, shared by build.yml and deploy.yml via jq -L; the tag rules are in .github/jq/tags.jq. Generated tags and PR matrix are identical to the previous ones. Co-Authored-By: Claude Sonnet 5.5 --- .github/jq/matrix.jq | 18 +++++ .github/jq/tags.jq | 35 ++++++++ .github/workflows/build.yml | 52 ++++++------ .github/workflows/deploy.yml | 153 ++++++++++++++--------------------- 4 files changed, 139 insertions(+), 119 deletions(-) create mode 100644 .github/jq/matrix.jq create mode 100644 .github/jq/tags.jq diff --git a/.github/jq/matrix.jq b/.github/jq/matrix.jq new file mode 100644 index 0000000..cb076b8 --- /dev/null +++ b/.github/jq/matrix.jq @@ -0,0 +1,18 @@ +# Build-matrix building blocks shared by build.yml (PR) and deploy.yml: +# jq -L .github/jq 'include "matrix"; ...' matrix.json +# The image variants (Dockerfile target), the platforms and the layer cache +# scope live here only, so the two workflows cannot drift apart. + +def targets: {std: "runner", ndk: "runner-ndk", slim: "runner-slim"}; + +def legs: { + amd64: {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest"}, + arm64: {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm"} +}; + +# One build leg. The standard image keeps the plain per-arch cache scope, the +# other variants their own (they also read the standard one). +def leg($java; $node; $variant; $arch): + {java_version: $java, node_version: $node, variant: $variant, target: targets[$variant]} + + legs[$arch] + | . + {cache_scope: (if $variant == "std" then $arch else "\($variant)-\($arch)" end)}; diff --git a/.github/jq/tags.jq b/.github/jq/tags.jq new file mode 100644 index 0000000..e286196 --- /dev/null +++ b/.github/jq/tags.jq @@ -0,0 +1,35 @@ +# Image tags of deploy.yml, one list per matrix entry: +# jq -L .github/jq 'include "matrix"; include "tags"; ...' matrix.json +# $v holds the Dockerfile versions: runner_version, compile_sdk, build_tools, +# ndk_version. $dj / $dn are the default JDK / Node major from matrix.json. + +# The fully pinned tag, baked into the image as RUNNER_IMAGE_TAG; it never +# changes meaning. +def image_tag($v): + if .variant == "slim" then "\($v.runner_version)-jdk\(.java_version)-node\(.node_version)-slim" + else "\($v.runner_version)-jdk\(.java_version)-sdk\($v.compile_sdk)-node\(.node_version)\(if .variant == "ndk" then "-ndk" else "" end)" + end; + +# Every tag the merged manifest list is published under. +def tags($v; $dj; $dn): + . as $e + | "\($v.runner_version)-jdk\($e.java_version)-sdk\($v.compile_sdk)" as $sdk + | if $e.variant == "std" then + [ (if $e.node_version == $dn then "jdk\($e.java_version)" else empty end), + (if $e.java_version == $dj then "node\($e.node_version)" else empty end), + "jdk\($e.java_version)-node\($e.node_version)", + "\($sdk)-node\($e.node_version)", + # Every Gradle-pinnable SDK component in the tag (compileSdk, build-tools), so + # consumers can let Renovate derive them from the image pin. + "\($sdk)-bt\($v.build_tools)-node\($e.node_version)", + (if $e.java_version == $dj and $e.node_version == $dn then "latest" else empty end) ] + elif $e.variant == "ndk" then + # Only the ndk variant ships an NDK, so only it spells the NDK version out + # (upstream's -sdk-ndk- and -sdk-bt-ndk- tags; Gradle ndkVersion via Renovate). + [ "ndk", + "\($sdk)-node\($e.node_version)-ndk", + "\($sdk)-ndk\($v.ndk_version)-node\($e.node_version)", + "\($sdk)-bt\($v.build_tools)-ndk\($v.ndk_version)-node\($e.node_version)" ] + else + ["slim", ($e | image_tag($v))] + end; diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d65f1a..c06f06f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,9 +51,26 @@ jobs: timeout-minutes: 5 outputs: matrix: ${{ steps.generate.outputs.matrix }} + runner_tag: ${{ steps.versions.outputs.runner_tag }} steps: - uses: actions/checkout@v7 + # Only VERSION is needed here: JDK and Node come from the matrix, and the + # SDK args are consumed straight from the Dockerfile defaults since + # nothing in a PR build needs them for tagging. + - name: Extract versions from Dockerfile + id: versions + run: | + set -euo pipefail + RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) + + if [ -z "$RUNNER_TAG" ]; then + echo "::error::Failed to extract versions from Dockerfile" + exit 1 + fi + + echo "runner_tag=${RUNNER_TAG}" >> "$GITHUB_OUTPUT" + - name: Generate matrix from matrix.json id: generate run: | @@ -67,20 +84,19 @@ jobs: # also gets the image smoke test (smoke: true), and the opt-in # runner-ndk and runner-slim targets get one amd64 build + smoke test # each, since deploy.yml publishes them. - MATRIX=$(jq -c ' + # The variants/legs table is shared with deploy.yml (.github/jq/matrix.jq). + MATRIX=$(jq -c -L .github/jq ' + include "matrix"; .default_jdk as $dj | .default_node as $dn | ( [ .java_versions[] | {java_version: ., node_version: $dn} ] + [ .node_versions[] | {java_version: $dj, node_version: .} ] | unique ) as $combos - | def leg($arch; $runner): {platform: "linux/\($arch)", arch: $arch, runner: $runner}; - def std: {variant: "std", target: "runner"}; - { include: - ( [ $combos[] | . + std + leg("amd64"; "ubuntu-latest") + | { include: + ( [ $combos[] | leg(.java_version; .node_version; "std"; "amd64") + {smoke: (.java_version == $dj and .node_version == $dn)} ] - + [ {java_version: $dj, node_version: $dn} + std + leg("arm64"; "ubuntu-24.04-arm") + {smoke: false} ] - + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "ndk", target: "runner-ndk"} ] - + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "slim", target: "runner-slim"} ] ) - | map(. + {cache_scope: (if .variant == "std" then .arch else "\(.variant)-\(.arch)" end)}) }' matrix.json) + + [ leg($dj; $dn; "std"; "arm64") + {smoke: false} ] + + [ leg($dj; $dn; "ndk"; "amd64") + {smoke: true} ] + + [ leg($dj; $dn; "slim"; "amd64") + {smoke: true} ] ) }' matrix.json) echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" build: @@ -94,22 +110,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Extract versions from Dockerfile - id: versions - run: | - set -euo pipefail - # Only VERSION is extracted here: JDK and Node come from the matrix, - # and the SDK args are consumed straight from the Dockerfile defaults - # since nothing in a PR build needs them for tagging. - RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) - - if [ -z "$RUNNER_TAG" ]; then - echo "::error::Failed to extract versions from Dockerfile" - exit 1 - fi - - echo "runner_tag=${RUNNER_TAG}" >> "$GITHUB_OUTPUT" - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -132,7 +132,7 @@ jobs: ${{ matrix.cache_scope != matrix.arch && format('type=gha,scope={0}', matrix.cache_scope) || '' }} cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} build-args: | - VERSION=${{ steps.versions.outputs.runner_tag }} + VERSION=${{ needs.matrix.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} NODE_VERSION=${{ matrix.node_version }} RUNNER_IMAGE_TAG=pr-${{ github.event.pull_request.number }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index caaf3de..b2afb88 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,36 +16,73 @@ jobs: outputs: build_matrix: ${{ steps.generate.outputs.build_matrix }} merge_matrix: ${{ steps.generate.outputs.merge_matrix }} + runner_tag: ${{ steps.versions.outputs.runner_tag }} steps: - uses: actions/checkout@v7 + # The only place the Dockerfile is read: the build jobs take the VERSION + # build-arg from this job's outputs, and every image tag is computed here, + # per matrix entry. + - name: Extract versions from Dockerfile + id: versions + run: | + set -euo pipefail + RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) + RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) + COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) + BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) + NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) + + if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then + echo "::error::Failed to extract versions from Dockerfile" + exit 1 + fi + + { + echo "runner_tag=${RUNNER_TAG}" + echo "runner_version=${RUNNER_VERSION}" + echo "compile_sdk=${COMPILE_SDK}" + echo "build_tools=${BUILD_TOOLS}" + echo "ndk_version=${NDK_VERSION}" + } >> "$GITHUB_OUTPUT" + - name: Generate matrices from matrix.json id: generate + env: + RUNNER_VERSION: ${{ steps.versions.outputs.runner_version }} + COMPILE_SDK: ${{ steps.versions.outputs.compile_sdk }} + BUILD_TOOLS: ${{ steps.versions.outputs.build_tools }} + NDK_VERSION: ${{ steps.versions.outputs.ndk_version }} run: | set -euo pipefail # Every JDK x Node pair is published as the `runner` target (variant # "std"). The default pair is additionally published as `runner-ndk` # (the image plus the NDK) and `runner-slim` (no Android SDK). Each # variant has its own digests, cache scope and tags so they never mix - # with the standard image. - ALL=$(jq -c ' - .default_jdk as $dj | .default_node as $dn - | [ .java_versions[] as $j | .node_versions[] as $n | - { java_version: $j, node_version: $n, - is_default_jdk: ($j == $dj), - is_default_node: ($n == $dn), - variant: "std", target: "runner" } ] as $std - | ( [ $std[] | select(.is_default_jdk and .is_default_node) ] ) as $default_pair - | ( [ $default_pair[] | . + {variant: "ndk", target: "runner-ndk"} ] - + [ $default_pair[] | . + {variant: "slim", target: "runner-slim"} ] ) as $extra - | ( $std + $extra ) as $combos + # with the standard image. The variant/platform/cache-scope table is + # shared with build.yml (.github/jq/matrix.jq); the tag rules are in + # .github/jq/tags.jq. A build entry carries the tag baked into the + # image, a merge entry the metadata-action `type=raw` lines of the + # manifest list. + ALL=$(jq -c -L .github/jq \ + --arg runner_version "$RUNNER_VERSION" --arg compile_sdk "$COMPILE_SDK" \ + --arg build_tools "$BUILD_TOOLS" --arg ndk_version "$NDK_VERSION" ' + include "matrix"; include "tags"; + {runner_version: $runner_version, compile_sdk: $compile_sdk, + build_tools: $build_tools, ndk_version: $ndk_version} as $v + | .default_jdk as $dj | .default_node as $dn + | [ .java_versions[] as $j | .node_versions[] as $n + | {java_version: $j, node_version: $n, variant: "std"} ] as $std + | ( $std + + [ $std[] | select(.java_version == $dj and .node_version == $dn) + | (. + {variant: "ndk"}), (. + {variant: "slim"}) ] ) as $images | { build: { include: [ - $combos[] as $c | - ($c + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest", - cache_scope: (if $c.variant == "std" then "amd64" else "\($c.variant)-amd64" end)}), - ($c + {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm", - cache_scope: (if $c.variant == "std" then "arm64" else "\($c.variant)-arm64" end)}) ] }, - merge: { include: $combos } }' matrix.json) + $images[] as $i | ("amd64", "arm64") as $arch + | leg($i.java_version; $i.node_version; $i.variant; $arch) + + {image_tag: ($i | image_tag($v))} ] }, + merge: { include: [ + $images[] + | . + {tags: (tags($v; $dj; $dn) | map("type=raw,value=" + .) | join("\n"))} ] } }' matrix.json) echo "build_matrix=$(jq -c .build <<< "$ALL")" >> "$GITHUB_OUTPUT" echo "merge_matrix=$(jq -c .merge <<< "$ALL")" >> "$GITHUB_OUTPUT" @@ -60,26 +97,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Extract versions from Dockerfile - id: versions - run: | - set -euo pipefail - RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) - RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) - COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) - - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ]; then - echo "::error::Failed to extract versions from Dockerfile" - exit 1 - fi - - # build_tools / ndk_version are only needed for the merge job's tags. - { - echo "runner_tag=${RUNNER_TAG}" - echo "runner_version=${RUNNER_VERSION}" - echo "compile_sdk=${COMPILE_SDK}" - } >> "$GITHUB_OUTPUT" - - name: Login to GitHub Container Registry uses: docker/login-action@v4 with: @@ -90,18 +107,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - # Image identity baked into the image as RUNNER_IMAGE_TAG / - # RUNNER_IMAGE_REVISION (Dockerfile ARG -> ENV). The tag is the fully - # pinned one, which never changes meaning; the revision is the commit. - - name: Docker metadata (image identity) - id: meta - uses: docker/metadata-action@v6 - with: - images: ghcr.io/${{ github.repository_owner }}/github-runner - tags: | - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}${{ matrix.variant == 'ndk' && '-ndk' || '' }},enable=${{ matrix.variant != 'slim' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} - - name: Build and push by digest id: build uses: docker/build-push-action@v7 @@ -121,11 +126,13 @@ jobs: # pipeline consumes the provenance, and turning it off keeps the # registry view to just amd64 + arm64. provenance: false + # RUNNER_IMAGE_TAG is the fully pinned tag, which never changes meaning; + # the revision is the commit. build-args: | - VERSION=${{ steps.versions.outputs.runner_tag }} + VERSION=${{ needs.matrix.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} NODE_VERSION=${{ matrix.node_version }} - RUNNER_IMAGE_TAG=${{ steps.meta.outputs.version }} + RUNNER_IMAGE_TAG=${{ matrix.image_tag }} RUNNER_IMAGE_REVISION=${{ github.sha }} # zstd layers pull and decompress faster on small hosts (Raspberry Pi) than gzip. # force-compression recompresses cached/base layers too; oci-mediatypes is @@ -156,30 +163,6 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.merge_matrix) }} steps: - - uses: actions/checkout@v7 - - - name: Extract versions from Dockerfile - id: versions - run: | - set -euo pipefail - RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) - RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) - COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) - BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) - NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then - echo "::error::Failed to extract versions from Dockerfile" - exit 1 - fi - - { - echo "runner_version=${RUNNER_VERSION}" - echo "compile_sdk=${COMPILE_SDK}" - echo "build_tools=${BUILD_TOOLS}" - echo "ndk_version=${NDK_VERSION}" - } >> "$GITHUB_OUTPUT" - - name: Download digests uses: actions/download-artifact@v8 with: @@ -202,23 +185,7 @@ jobs: uses: docker/metadata-action@v6 with: images: ghcr.io/${{ github.repository_owner }}/github-runner - tags: | - type=raw,value=jdk${{ matrix.java_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_node }} - type=raw,value=node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_jdk }} - type=raw,value=jdk${{ matrix.java_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} - # Every Gradle-pinnable SDK component in the tag (compileSdk, build-tools), so - # consumers can let Renovate derive them from the image pin. - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} - type=raw,value=latest,enable=${{ matrix.variant == 'std' && matrix.is_default_jdk && matrix.is_default_node }} - type=raw,value=ndk,enable=${{ matrix.variant == 'ndk' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}-ndk,enable=${{ matrix.variant == 'ndk' }} - # Only the ndk variant ships an NDK, so only it spells the NDK version out - # (upstream's -sdk-ndk- and -sdk-bt-ndk- tags; Gradle ndkVersion via Renovate). - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} - type=raw,value=slim,enable=${{ matrix.variant == 'slim' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} + tags: ${{ matrix.tags }} - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests From 7e0c21aa0c0156cc59a9c3461fa9a9719f9403fa Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:27:25 +0300 Subject: [PATCH 3/3] test: split the harness from the pruner fixtures and share it across tests tests/harness.sh holds the pure harness (scratch dir with exit trap, fail/gone/kept/finish, at_exit); tests/lib.sh keeps only the pruner fixtures on top of it, so tests/pruner-*.sh are unchanged. slots.sh and image-smoke.sh source the harness instead of keeping their own fail counters. image-smoke.sh runs its checks with docker exec in one idle container instead of a docker run per check. CI skips harness.sh and lib.sh in the test loop and shellchecks tests/bench. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/build.yml | 3 ++- tests/harness.sh | 36 +++++++++++++++++++++++++++++++++ tests/image-smoke.sh | 19 +++++++++--------- tests/lib.sh | 30 ++++++++-------------------- tests/slots.sh | 40 +++++++++++++++---------------------- 5 files changed, 72 insertions(+), 56 deletions(-) create mode 100644 tests/harness.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c06f06f..767f504 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,8 @@ jobs: set -euo pipefail for t in tests/*.sh; do # image-smoke.sh needs a built image; it runs in the build job. - [ "$(basename "$t")" = image-smoke.sh ] && continue + # harness.sh and lib.sh are sourced by the tests, not tests. + case "$(basename "$t")" in image-smoke.sh | harness.sh | lib.sh) continue ;; esac bash "$t" done diff --git a/tests/harness.sh b/tests/harness.sh new file mode 100644 index 0000000..a25164f --- /dev/null +++ b/tests/harness.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Test harness shared by tests/*.sh. Sourced, not run. Gives a scratch dir +# ($work, removed on exit), check bookkeeping (fail / gone / kept / finish) and +# at_exit for extra cleanup. Knows nothing about what is under test; the pruner +# fixtures live in tests/lib.sh. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # tests/ +# shellcheck disable=SC2034 # used by the tests that source this +root="$(cd "$here/.." && pwd)" # repo root +work="$(mktemp -d "${TMPDIR:-/tmp}/test.XXXXXX")" + +cleanups=() +# at_exit : run it when the test ends, before $work is removed. +at_exit() { cleanups+=("$*"); } +run_cleanups() { + local c + for c in "${cleanups[@]}"; do eval "$c" || true; done + rm -rf "$work" +} +trap run_cleanups EXIT + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } + +gone() { if [ -e "$1" ] || [ -L "$1" ]; then fail "$1 should be evicted"; fi; } +kept() { [ -e "$1" ] || [ -L "$1" ] || fail "$1 should survive"; } + +# finish : end of a test; fails the run when a check failed. +finish() { + if [ "$fails" -gt 0 ]; then + echo "$fails check(s) failed" + exit 1 + fi + echo "PASS: $1" +} diff --git a/tests/image-smoke.sh b/tests/image-smoke.sh index d00ee6d..961442a 100755 --- a/tests/image-smoke.sh +++ b/tests/image-smoke.sh @@ -6,7 +6,9 @@ # (runner-slim) must carry java/node/tools but no Android SDK. # The check snippets are single-quoted on purpose: they expand inside the image. # shellcheck disable=SC2016 -set -euo pipefail + +# shellcheck source=tests/harness.sh +source "$(dirname "${BASH_SOURCE[0]}")/harness.sh" image="${1:?usage: tests/image-smoke.sh [jdk-major] [node-major] [std|ndk|slim]}" want_jdk="${2:-}" @@ -17,13 +19,16 @@ case "$variant" in *) echo "unknown variant '$variant' (want std, ndk or slim)" >&2; exit 2 ;; esac -fails=0 -fail() { echo "FAIL: $*"; fails=$((fails + 1)); } pass() { echo "ok: $*"; } indent() { printf ' %s\n' "${1//$'\n'/$'\n '}"; } +# One container, kept idle, runs every check: a docker run per check cost +# ~0.5 s of startup each. +container="$(docker run -d --rm --entrypoint sleep "$image" infinity)" +at_exit "docker rm -f $container >/dev/null 2>&1" + # in_image