diff --git a/.github/jq/matrix.jq b/.github/jq/matrix.jq new file mode 100644 index 0000000..cb076b8 --- /dev/null +++ b/.github/jq/matrix.jq @@ -0,0 +1,18 @@ +# Build-matrix building blocks shared by build.yml (PR) and deploy.yml: +# jq -L .github/jq 'include "matrix"; ...' matrix.json +# The image variants (Dockerfile target), the platforms and the layer cache +# scope live here only, so the two workflows cannot drift apart. + +def targets: {std: "runner", ndk: "runner-ndk", slim: "runner-slim"}; + +def legs: { + amd64: {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest"}, + arm64: {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm"} +}; + +# One build leg. The standard image keeps the plain per-arch cache scope, the +# other variants their own (they also read the standard one). +def leg($java; $node; $variant; $arch): + {java_version: $java, node_version: $node, variant: $variant, target: targets[$variant]} + + legs[$arch] + | . + {cache_scope: (if $variant == "std" then $arch else "\($variant)-\($arch)" end)}; diff --git a/.github/jq/tags.jq b/.github/jq/tags.jq new file mode 100644 index 0000000..e286196 --- /dev/null +++ b/.github/jq/tags.jq @@ -0,0 +1,35 @@ +# Image tags of deploy.yml, one list per matrix entry: +# jq -L .github/jq 'include "matrix"; include "tags"; ...' matrix.json +# $v holds the Dockerfile versions: runner_version, compile_sdk, build_tools, +# ndk_version. $dj / $dn are the default JDK / Node major from matrix.json. + +# The fully pinned tag, baked into the image as RUNNER_IMAGE_TAG; it never +# changes meaning. +def image_tag($v): + if .variant == "slim" then "\($v.runner_version)-jdk\(.java_version)-node\(.node_version)-slim" + else "\($v.runner_version)-jdk\(.java_version)-sdk\($v.compile_sdk)-node\(.node_version)\(if .variant == "ndk" then "-ndk" else "" end)" + end; + +# Every tag the merged manifest list is published under. +def tags($v; $dj; $dn): + . as $e + | "\($v.runner_version)-jdk\($e.java_version)-sdk\($v.compile_sdk)" as $sdk + | if $e.variant == "std" then + [ (if $e.node_version == $dn then "jdk\($e.java_version)" else empty end), + (if $e.java_version == $dj then "node\($e.node_version)" else empty end), + "jdk\($e.java_version)-node\($e.node_version)", + "\($sdk)-node\($e.node_version)", + # Every Gradle-pinnable SDK component in the tag (compileSdk, build-tools), so + # consumers can let Renovate derive them from the image pin. + "\($sdk)-bt\($v.build_tools)-node\($e.node_version)", + (if $e.java_version == $dj and $e.node_version == $dn then "latest" else empty end) ] + elif $e.variant == "ndk" then + # Only the ndk variant ships an NDK, so only it spells the NDK version out + # (upstream's -sdk-ndk- and -sdk-bt-ndk- tags; Gradle ndkVersion via Renovate). + [ "ndk", + "\($sdk)-node\($e.node_version)-ndk", + "\($sdk)-ndk\($v.ndk_version)-node\($e.node_version)", + "\($sdk)-bt\($v.build_tools)-ndk\($v.ndk_version)-node\($e.node_version)" ] + else + ["slim", ($e | image_tag($v))] + end; diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 783e42b..767f504 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: # nullglob: a pattern that matches nothing (e.g. hosts/*.sh) must # drop out instead of being passed to shellcheck as a literal. shopt -s nullglob - files=(*.sh hosts/*.sh tests/*.sh) + files=(*.sh hosts/*.sh tests/*.sh tests/bench/*.sh) if [ "${#files[@]}" -eq 0 ]; then echo "::error::No shell scripts found to check" exit 1 @@ -41,7 +41,8 @@ jobs: set -euo pipefail for t in tests/*.sh; do # image-smoke.sh needs a built image; it runs in the build job. - [ "$(basename "$t")" = image-smoke.sh ] && continue + # harness.sh and lib.sh are sourced by the tests, not tests. + case "$(basename "$t")" in image-smoke.sh | harness.sh | lib.sh) continue ;; esac bash "$t" done @@ -51,9 +52,26 @@ jobs: timeout-minutes: 5 outputs: matrix: ${{ steps.generate.outputs.matrix }} + runner_tag: ${{ steps.versions.outputs.runner_tag }} steps: - uses: actions/checkout@v7 + # Only VERSION is needed here: JDK and Node come from the matrix, and the + # SDK args are consumed straight from the Dockerfile defaults since + # nothing in a PR build needs them for tagging. + - name: Extract versions from Dockerfile + id: versions + run: | + set -euo pipefail + RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) + + if [ -z "$RUNNER_TAG" ]; then + echo "::error::Failed to extract versions from Dockerfile" + exit 1 + fi + + echo "runner_tag=${RUNNER_TAG}" >> "$GITHUB_OUTPUT" + - name: Generate matrix from matrix.json id: generate run: | @@ -67,20 +85,19 @@ jobs: # also gets the image smoke test (smoke: true), and the opt-in # runner-ndk and runner-slim targets get one amd64 build + smoke test # each, since deploy.yml publishes them. - MATRIX=$(jq -c ' + # The variants/legs table is shared with deploy.yml (.github/jq/matrix.jq). + MATRIX=$(jq -c -L .github/jq ' + include "matrix"; .default_jdk as $dj | .default_node as $dn | ( [ .java_versions[] | {java_version: ., node_version: $dn} ] + [ .node_versions[] | {java_version: $dj, node_version: .} ] | unique ) as $combos - | def leg($arch; $runner): {platform: "linux/\($arch)", arch: $arch, runner: $runner}; - def std: {variant: "std", target: "runner"}; - { include: - ( [ $combos[] | . + std + leg("amd64"; "ubuntu-latest") + | { include: + ( [ $combos[] | leg(.java_version; .node_version; "std"; "amd64") + {smoke: (.java_version == $dj and .node_version == $dn)} ] - + [ {java_version: $dj, node_version: $dn} + std + leg("arm64"; "ubuntu-24.04-arm") + {smoke: false} ] - + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "ndk", target: "runner-ndk"} ] - + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "slim", target: "runner-slim"} ] ) - | map(. + {cache_scope: (if .variant == "std" then .arch else "\(.variant)-\(.arch)" end)}) }' matrix.json) + + [ leg($dj; $dn; "std"; "arm64") + {smoke: false} ] + + [ leg($dj; $dn; "ndk"; "amd64") + {smoke: true} ] + + [ leg($dj; $dn; "slim"; "amd64") + {smoke: true} ] ) }' matrix.json) echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" build: @@ -94,22 +111,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Extract versions from Dockerfile - id: versions - run: | - set -euo pipefail - # Only VERSION is extracted here: JDK and Node come from the matrix, - # and the SDK args are consumed straight from the Dockerfile defaults - # since nothing in a PR build needs them for tagging. - RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) - - if [ -z "$RUNNER_TAG" ]; then - echo "::error::Failed to extract versions from Dockerfile" - exit 1 - fi - - echo "runner_tag=${RUNNER_TAG}" >> "$GITHUB_OUTPUT" - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -132,7 +133,7 @@ jobs: ${{ matrix.cache_scope != matrix.arch && format('type=gha,scope={0}', matrix.cache_scope) || '' }} cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} build-args: | - VERSION=${{ steps.versions.outputs.runner_tag }} + VERSION=${{ needs.matrix.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} NODE_VERSION=${{ matrix.node_version }} RUNNER_IMAGE_TAG=pr-${{ github.event.pull_request.number }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index caaf3de..b2afb88 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,36 +16,73 @@ jobs: outputs: build_matrix: ${{ steps.generate.outputs.build_matrix }} merge_matrix: ${{ steps.generate.outputs.merge_matrix }} + runner_tag: ${{ steps.versions.outputs.runner_tag }} steps: - uses: actions/checkout@v7 + # The only place the Dockerfile is read: the build jobs take the VERSION + # build-arg from this job's outputs, and every image tag is computed here, + # per matrix entry. + - name: Extract versions from Dockerfile + id: versions + run: | + set -euo pipefail + RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) + RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) + COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) + BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) + NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) + + if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then + echo "::error::Failed to extract versions from Dockerfile" + exit 1 + fi + + { + echo "runner_tag=${RUNNER_TAG}" + echo "runner_version=${RUNNER_VERSION}" + echo "compile_sdk=${COMPILE_SDK}" + echo "build_tools=${BUILD_TOOLS}" + echo "ndk_version=${NDK_VERSION}" + } >> "$GITHUB_OUTPUT" + - name: Generate matrices from matrix.json id: generate + env: + RUNNER_VERSION: ${{ steps.versions.outputs.runner_version }} + COMPILE_SDK: ${{ steps.versions.outputs.compile_sdk }} + BUILD_TOOLS: ${{ steps.versions.outputs.build_tools }} + NDK_VERSION: ${{ steps.versions.outputs.ndk_version }} run: | set -euo pipefail # Every JDK x Node pair is published as the `runner` target (variant # "std"). The default pair is additionally published as `runner-ndk` # (the image plus the NDK) and `runner-slim` (no Android SDK). Each # variant has its own digests, cache scope and tags so they never mix - # with the standard image. - ALL=$(jq -c ' - .default_jdk as $dj | .default_node as $dn - | [ .java_versions[] as $j | .node_versions[] as $n | - { java_version: $j, node_version: $n, - is_default_jdk: ($j == $dj), - is_default_node: ($n == $dn), - variant: "std", target: "runner" } ] as $std - | ( [ $std[] | select(.is_default_jdk and .is_default_node) ] ) as $default_pair - | ( [ $default_pair[] | . + {variant: "ndk", target: "runner-ndk"} ] - + [ $default_pair[] | . + {variant: "slim", target: "runner-slim"} ] ) as $extra - | ( $std + $extra ) as $combos + # with the standard image. The variant/platform/cache-scope table is + # shared with build.yml (.github/jq/matrix.jq); the tag rules are in + # .github/jq/tags.jq. A build entry carries the tag baked into the + # image, a merge entry the metadata-action `type=raw` lines of the + # manifest list. + ALL=$(jq -c -L .github/jq \ + --arg runner_version "$RUNNER_VERSION" --arg compile_sdk "$COMPILE_SDK" \ + --arg build_tools "$BUILD_TOOLS" --arg ndk_version "$NDK_VERSION" ' + include "matrix"; include "tags"; + {runner_version: $runner_version, compile_sdk: $compile_sdk, + build_tools: $build_tools, ndk_version: $ndk_version} as $v + | .default_jdk as $dj | .default_node as $dn + | [ .java_versions[] as $j | .node_versions[] as $n + | {java_version: $j, node_version: $n, variant: "std"} ] as $std + | ( $std + + [ $std[] | select(.java_version == $dj and .node_version == $dn) + | (. + {variant: "ndk"}), (. + {variant: "slim"}) ] ) as $images | { build: { include: [ - $combos[] as $c | - ($c + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest", - cache_scope: (if $c.variant == "std" then "amd64" else "\($c.variant)-amd64" end)}), - ($c + {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm", - cache_scope: (if $c.variant == "std" then "arm64" else "\($c.variant)-arm64" end)}) ] }, - merge: { include: $combos } }' matrix.json) + $images[] as $i | ("amd64", "arm64") as $arch + | leg($i.java_version; $i.node_version; $i.variant; $arch) + + {image_tag: ($i | image_tag($v))} ] }, + merge: { include: [ + $images[] + | . + {tags: (tags($v; $dj; $dn) | map("type=raw,value=" + .) | join("\n"))} ] } }' matrix.json) echo "build_matrix=$(jq -c .build <<< "$ALL")" >> "$GITHUB_OUTPUT" echo "merge_matrix=$(jq -c .merge <<< "$ALL")" >> "$GITHUB_OUTPUT" @@ -60,26 +97,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Extract versions from Dockerfile - id: versions - run: | - set -euo pipefail - RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) - RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) - COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) - - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ]; then - echo "::error::Failed to extract versions from Dockerfile" - exit 1 - fi - - # build_tools / ndk_version are only needed for the merge job's tags. - { - echo "runner_tag=${RUNNER_TAG}" - echo "runner_version=${RUNNER_VERSION}" - echo "compile_sdk=${COMPILE_SDK}" - } >> "$GITHUB_OUTPUT" - - name: Login to GitHub Container Registry uses: docker/login-action@v4 with: @@ -90,18 +107,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - # Image identity baked into the image as RUNNER_IMAGE_TAG / - # RUNNER_IMAGE_REVISION (Dockerfile ARG -> ENV). The tag is the fully - # pinned one, which never changes meaning; the revision is the commit. - - name: Docker metadata (image identity) - id: meta - uses: docker/metadata-action@v6 - with: - images: ghcr.io/${{ github.repository_owner }}/github-runner - tags: | - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}${{ matrix.variant == 'ndk' && '-ndk' || '' }},enable=${{ matrix.variant != 'slim' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} - - name: Build and push by digest id: build uses: docker/build-push-action@v7 @@ -121,11 +126,13 @@ jobs: # pipeline consumes the provenance, and turning it off keeps the # registry view to just amd64 + arm64. provenance: false + # RUNNER_IMAGE_TAG is the fully pinned tag, which never changes meaning; + # the revision is the commit. build-args: | - VERSION=${{ steps.versions.outputs.runner_tag }} + VERSION=${{ needs.matrix.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} NODE_VERSION=${{ matrix.node_version }} - RUNNER_IMAGE_TAG=${{ steps.meta.outputs.version }} + RUNNER_IMAGE_TAG=${{ matrix.image_tag }} RUNNER_IMAGE_REVISION=${{ github.sha }} # zstd layers pull and decompress faster on small hosts (Raspberry Pi) than gzip. # force-compression recompresses cached/base layers too; oci-mediatypes is @@ -156,30 +163,6 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.merge_matrix) }} steps: - - uses: actions/checkout@v7 - - - name: Extract versions from Dockerfile - id: versions - run: | - set -euo pipefail - RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) - RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) - COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) - BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) - NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then - echo "::error::Failed to extract versions from Dockerfile" - exit 1 - fi - - { - echo "runner_version=${RUNNER_VERSION}" - echo "compile_sdk=${COMPILE_SDK}" - echo "build_tools=${BUILD_TOOLS}" - echo "ndk_version=${NDK_VERSION}" - } >> "$GITHUB_OUTPUT" - - name: Download digests uses: actions/download-artifact@v8 with: @@ -202,23 +185,7 @@ jobs: uses: docker/metadata-action@v6 with: images: ghcr.io/${{ github.repository_owner }}/github-runner - tags: | - type=raw,value=jdk${{ matrix.java_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_node }} - type=raw,value=node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_jdk }} - type=raw,value=jdk${{ matrix.java_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} - # Every Gradle-pinnable SDK component in the tag (compileSdk, build-tools), so - # consumers can let Renovate derive them from the image pin. - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} - type=raw,value=latest,enable=${{ matrix.variant == 'std' && matrix.is_default_jdk && matrix.is_default_node }} - type=raw,value=ndk,enable=${{ matrix.variant == 'ndk' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}-ndk,enable=${{ matrix.variant == 'ndk' }} - # Only the ndk variant ships an NDK, so only it spells the NDK version out - # (upstream's -sdk-ndk- and -sdk-bt-ndk- tags; Gradle ndkVersion via Renovate). - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} - type=raw,value=slim,enable=${{ matrix.variant == 'slim' }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} + tags: ${{ matrix.tags }} - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests diff --git a/pruner.sh b/pruner.sh index 17f0681..28c0f18 100755 --- a/pruner.sh +++ b/pruner.sh @@ -66,7 +66,7 @@ DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } -# Evictable units per store, one rule per line: store|kind|glob|skip +# Evictable units per store, one rule per line: store|kind|glob|skip|opts # kind d: every directory matching glob (relative to the store dir, never a # symlink) is one unit, removed whole; skip is a case pattern for the # relative path of units to leave alone. Glob "." makes the whole @@ -74,40 +74,44 @@ log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } # emptied, but its subdirs are kept, as container symlinks point there. # kind f: every regular file named skip (a find -name pattern) under the dir # glob is one unit. +# opts: comma-separated. shared: not a runner-slot store (the action archive +# cache). key=: LRU order of a dir unit is the atime of / +# (the unit's mtime when it has none), not the newest atime of any file. +# Globs use * and ? only; like the shell, they skip names starting with a dot. # Anything no rule matches (bun's bin/ and @t@ piles, cargo's bin/ and registry -# index, rustup's settings.toml, ...) is never touched. Hooks for the few stores -# needing more than a rule are in unit_key / unit_protected / unit_pre_evict / -# unit_post_evict. Leftovers of killed bun extractions (*.tmp, *.part) are -# pre-job.sh's job: it sweeps them before the next job of a crashed slot. +# index, rustup's settings.toml, ...) is never touched. The few stores needing +# more than a rule are in store_hook. Leftovers of killed bun extractions +# (*.tmp, *.part) are pre-job.sh's job: it sweeps them before the next job of a +# crashed slot. STORE_TABLE=' -bun|d|install/cache/*@*|install/cache/@* -bun|d|install/cache/@*/*@*|install/cache/@t@/* -npm|f|_cacache|* -pnpm|d|.| -gradle|d|caches/modules-2/files-2.1/*/*/*| -gradle|d|caches/transforms-*/*| -gradle|d|wrapper/dists/*| -gradle|d|caches/jars-*/*| -gradle|f|caches/build-cache-1|* -cargo|f|registry/cache|*.crate -cargo|d|registry/src/*/*| -cargo|d|git/checkouts/*| -cargo|d|git/db/*| -rustup|d|toolchains/*| -konan|d|dependencies/*| -konan|d|kotlin-native-prebuilt-*| -mise-data|d|installs/*/*| -pub|d|hosted/*/*| -pub|d|git/*| -toolcache|d|*/*| -mise-cache|f|.|* -hk|f|.|* -actions|f|.|* +bun|d|install/cache/*@*|install/cache/@*|key=package.json +bun|d|install/cache/@*/*@*|install/cache/@t@/*|key=package.json +npm|f|_cacache|*| +pnpm|d|.|| +gradle|d|caches/modules-2/files-2.1/*/*/*|| +gradle|d|caches/transforms-*/*|| +gradle|d|wrapper/dists/*|| +gradle|d|caches/jars-*/*|| +gradle|f|caches/build-cache-1|*| +cargo|f|registry/cache|*.crate| +cargo|d|registry/src/*/*|| +cargo|d|git/checkouts/*|| +cargo|d|git/db/*|| +rustup|d|toolchains/*|| +konan|d|dependencies/*|| +konan|d|kotlin-native-prebuilt-*|| +mise-data|d|installs/*/*|| +pub|d|hosted/*/*|| +pub|d|git/*|| +toolcache|d|*/*|| +mise-cache|f|.|*| +hk|f|.|*| +actions|f|.|*|shared ' # Stores of a runner slot (everything but the shared action archive cache). slot_stores() { - awk -F'|' 'NF > 3 && $1 != "actions" && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" + awk -F'|' 'NF > 3 && $5 !~ /(^|,)shared(,|$)/ && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" } dir_bytes() { @@ -119,57 +123,132 @@ rustup_default() { sed -n 's/^default_toolchain[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1/settings.toml" 2>/dev/null | head -n 1 } +# The per-store behaviour a table rule cannot express: store_hook . +# protect prints the names of units never to evict, one +# per line (a unit named N or N-* is kept) +# pre before a dir unit is removed +# post after it is gone +store_hook() { + local base + case "$1:$2" in + # `stable` is the dir stable-, hence the N-* match. + protect:rustup) rustup_default "$3" ;; + # /.complete says / is whole. Drop the markers first + # so an interrupted removal never leaves a "complete" half tree. + pre:toolcache) find "$3" -mindepth 1 -maxdepth 1 -name '*.complete' -delete 2>/dev/null ;; + # Drop the /@@@ symlink pointing at the unit, and the dir if empty. + post:bun) + base="${3##*/}" + rm -f "${3%/*}/${base%%@*}/${base#*@}" 2>/dev/null + rmdir "${3%/*}/${base%%@*}" 2>/dev/null + ;; + esac + return 0 +} + # bun treats install/cache/@@@@/ as complete whenever the folder # exists, so a file evicted from it is silently missing from every later # node_modules (next@16.3.8 lost dist/compiled/server-only/empty.js, breaking # every Next build). 0-byte files went first: copying them never reads, so # their atime never moved. Hence whole packages, least recently used first by -# the atime of their package.json (read on every install and import). +# the atime of their package.json (read on every install and import): the +# key=package.json option of the bun rules. -# LRU key override for dir unit $2 of store $1; prints nothing to keep the -# newest-atime-of-any-file key. -unit_key() { - [ "$1" = bun ] || return 0 - stat -c %X "$2/package.json" 2>/dev/null || stat -c %Y "$2" 2>/dev/null +# awk program of collect_units. Input: find lines "%P\t%y\t%A@\t%T@\t%s\t%d" +# (path below the rule's prefix, type, atime, mtime, size, depth). Variables: +# store, sd (store dir), pre (rule prefix), glob (rule glob below pre), depth +# (its component count), skip (case pattern on the path below sd), keyfile (see +# key= in STORE_TABLE), protect (names never evicted, one per line). +# mawk compatible (this is Ubuntu's default awk): no gensub, no NUL in strings. +# shellcheck disable=SC2016 # awk program, not shell +UNITS_AWK=' +# Shell glob (* and ? only; a leading dot is never matched) as an anchored regex. +function glob_re(g, i, ch, out, start) { + out = "^"; start = 1 + for (i = 1; i <= length(g); i++) { + ch = substr(g, i, 1) + if (ch == "*") out = out (start ? "([^./][^/]*)?" : "[^/]*") + else if (ch == "?") out = out (start ? "[^./]" : "[^/]") + else if (index(".+(){}|^$[]\\", ch)) out = out "\\" ch + else out = out ch + start = (ch == "/") + } + return out "$" } - -# Succeeds when unit $3 (relative to dir $2 of store $1) must not be evicted. -unit_protected() { - local def - [ "$1" = rustup ] || return 1 - def=$(rustup_default "$2") - [ -n "$def" ] || return 1 - case "${3##*/}" in - "$def" | "$def"-*) return 0 ;; # `stable` is the dir stable- - esac - return 1 +# `case` pattern (* matches / too) as an anchored regex. +function case_re(g, i, ch, out) { + out = "^" + for (i = 1; i <= length(g); i++) { + ch = substr(g, i, 1) + if (ch == "*") out = out ".*" + else if (ch == "?") out = out "." + else if (index(".+(){}|^$[]\\", ch)) out = out "\\" ch + else out = out ch + } + return out "$" } - -# toolcache: /.complete says / is whole. Drop the markers -# first so an interrupted removal never leaves a "complete" half tree. -unit_pre_evict() { - [ "$1" = toolcache ] || return 0 - find "$2" -mindepth 1 -maxdepth 1 -name '*.complete' -delete 2>/dev/null +function flush( key) { + if (cur == "") return + key = keyfile == "" ? maxf : (keyat != "" ? keyat : mtime) + printf "%.6f\t%.0f\t%s\t%s\t%s%c", key, size, store, stop, sd "/" rel, 0 + cur = "" } - -# After unit $2 of store $1 is gone. -unit_post_evict() { - local base - [ "$1" = bun ] || return 0 - # Drop the /@@@ symlink pointing at it, and the dir if empty. - base="${2##*/}" - rm -f "${2%/*}/${base%%@*}/${base#*@}" 2>/dev/null - rmdir "${2%/*}/${base%%@*}" 2>/dev/null +function protected(rel, base, i) { + base = rel; sub(/^.*\//, "", base) + for (i = 1; i <= nprot; i++) + if (prot[i] != "" && (base == prot[i] || index(base, prot[i] "-") == 1)) return 1 return 0 } +BEGIN { + FS = "\t" + root = (glob == ".") + if (root) depth = 0 + gre = glob_re(glob) + sre = skip == "" ? "" : case_re(skip) + nprot = split(protect, prot, "\n") + stop = sd (pre == "" ? "" : "/" pre) +} +NF != 6 { next } +{ + d = $6 + 0 + if (d > depth) { # below the unit level: part of the current unit, if any + if (cur == "") next + if (!root && substr($1, 1, plen) != cp) next + size += $5 + if ($2 == "f") { + if ($3 + 0 > maxf) maxf = $3 + 0 + if (keyfile != "" && $1 == cp keyfile) keyat = $3 + } + next + } + flush() + if (d < depth || $2 != "d") next + if (!root && $1 !~ gre) next + rel = root ? "." : (pre == "" ? $1 : pre "/" $1) + if (sre != "" && rel ~ sre) next + if (protected(rel)) next + cur = rel; cp = $1 "/"; plen = length(cp) + if (root) cp = "" + size = $5; maxf = 0; keyat = ""; mtime = $4 +} +END { flush() } +' # Print NUL-terminated "keysizestorestoppath" records, one # per evictable unit of store $1 living in dir $2. key is the LRU order (atime, # seconds); stop is the dir above which emptied parents are not removed. +# +# One find per dir rule lists everything below the rule's fixed prefix, and one +# awk turns that listing into units. A unit sits at a fixed depth (the glob's +# component count) and find lists a directory right before its contents, so a +# unit's size, newest file atime and package.json atime are summed up as the +# lines go by: no per-unit process. collect_units() { - local store="$1" sd="$2" s kind glob skip pre stop + local store="$1" sd="$2" s kind glob skip opts pre rest key protect + local -a parts [ -d "$sd" ] || return 0 - while IFS='|' read -r s kind glob skip; do + protect=$(store_hook protect "$store" "$sd") + while IFS='|' read -r s kind glob skip opts; do [ "$s" = "$store" ] || continue if [ "$kind" = f ]; then [ -d "$sd/$glob" ] || continue @@ -181,24 +260,15 @@ collect_units() { */*) pre="${pre%/*}" ;; *) pre="" ;; esac - stop="$sd${pre:+/$pre}" - ( - cd "$sd" 2>/dev/null || exit 0 - shopt -s nullglob - # shellcheck disable=SC2086 # glob expansion is the point - for rel in $glob; do - if [ ! -d "$rel" ] || [ -L "$rel" ]; then continue; fi - # shellcheck disable=SC2254 # skip is a pattern - case "$rel" in $skip) continue ;; esac - if unit_protected "$store" "$sd" "$rel"; then continue; fi - stats=$(find "$rel" -printf '%y\t%A@\t%s\n' 2>/dev/null | - awk -F'\t' '{ s += $3; if ($1 == "f" && $2 + 0 > m) m = $2 + 0 } END { printf "%.6f\t%.0f\n", m, s }') - key="${stats%%$'\t'*}" - t=$(unit_key "$store" "$rel") - [ -z "$t" ] || key="$t" - printf '%s\t%s\t%s\t%s\t%s\0' "$key" "${stats#*$'\t'}" "$store" "$stop" "$sd/$rel" - done - ) + rest="${glob#"$pre"}" + rest="${rest#/}" + IFS=/ read -ra parts <<<"$rest" + key="" + case ",$opts," in *,key=*) key="${opts#*key=}"; key="${key%%,*}" ;; esac + [ -d "$sd${pre:+/$pre}" ] || continue + find "$sd${pre:+/$pre}" -printf '%P\t%y\t%A@\t%T@\t%s\t%d\n' 2>/dev/null | + awk -F'\t' -v store="$store" -v sd="$sd" -v pre="$pre" -v glob="$rest" -v depth="${#parts[@]}" \ + -v skip="$skip" -v keyfile="$key" -v protect="$protect" "$UNITS_AWK" done <<<"$STORE_TABLE" } @@ -210,9 +280,9 @@ evict_unit() { find "${path%/.}" -mindepth 2 -delete 2>/dev/null find "${path%/.}" -mindepth 1 -maxdepth 1 ! -type d -delete 2>/dev/null elif [ -d "$path" ] && [ ! -L "$path" ]; then - unit_pre_evict "$store" "$path" + store_hook pre "$store" "$path" rm -rf "$path" 2>/dev/null - unit_post_evict "$store" "$path" + store_hook post "$store" "$path" p="${path%/*}" while [ "$p" != "$stop" ] && [ "${p#"$stop"/}" != "$p" ]; do rmdir "$p" 2>/dev/null || break diff --git a/tests/bench/pruner-100k.sh b/tests/bench/pruner-100k.sh new file mode 100755 index 0000000..37db701 --- /dev/null +++ b/tests/bench/pruner-100k.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Timing of one pruner pass over a synthetic cache of ~100k evictable units +# (50k bun packages, 30k gradle module versions, 20k npm cache files), with a +# budget at 90% of its size so about a tenth gets evicted. Not a test: CI only +# runs tests/*.sh. Needs GNU tools; on macOS: +# docker run --rm -v "$PWD:/w" -w /w ubuntu:24.04 bash tests/bench/pruner-100k.sh [scale-percent] +# Prints the unit count, the time of the collect phase alone, and of a full +# prune_caches pass (collect + sort + evict). +set -uo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +scale="${1:-100}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# shellcheck source=pruner.sh +source "${PRUNER_SH:-$here/pruner.sh}" +RUNNER_DATA_DIR="$work/data" +ACTION_CACHE_DIR="$work/no-action-cache" +CACHE_BUDGET_PCT=100 + +# mkfiles : read dirs on stdin, create each with a file in it, +# atimes spread over 50 ages. +mkfiles() { + local file="$1" list="$work/list" k + cat >"$list" + xargs -r -a "$list" -n 2000 -P 4 mkdir -p + for k in $(seq 0 49); do + awk -v k="$k" -v f="$file" 'NR % 50 == k {print $0 "/" f}' "$list" >"$work/files" + # shellcheck disable=SC2016 # expands in the child shell + xargs -r -a "$work/files" -n 2000 sh -c 'for p; do printf "%1024s" x >"$p"; done' _ + xargs -r -a "$work/files" -n 2000 touch -a -d "$((k + 1)) days ago" + done +} + +slot="$RUNNER_DATA_DIR/slots/1" +cache="$slot/cache" +mkdir -p "$cache" "$slot/work" + +seq 1 $((50000 * scale / 100)) | + awk -v d="$cache/bun/install/cache" '{print d "/pkg" $1 "@1.0.0@@@1"}' | mkfiles package.json +seq 1 $((30000 * scale / 100)) | + awk -v d="$cache/gradle/caches/modules-2/files-2.1" '{print d "/org.g" $1 "/lib/1.0.0/h"}' | mkfiles lib.jar +seq 1 $((20000 * scale / 100)) | + awk -v d="$cache/npm/_cacache" '{print d "/c" $1}' | mkfiles f +echo "fixture: $(find "$cache" -type f | wc -l) files" + +CACHE_BUDGET_BYTES=$(($(dir_bytes "$cache") * 9 / 10)) + +start=$EPOCHREALTIME +units=$(for s in bun gradle npm; do collect_units "$s" "$cache/$s"; done | tr -cd '\0' | wc -c) +end=$EPOCHREALTIME +echo "units: $units" +echo "collect: $(awk -v a="$start" -v b="$end" 'BEGIN { printf "%.1f", b - a }') s" + +start=$EPOCHREALTIME +prune_caches >"$work/prune.log" 2>&1; rc=$? +end=$EPOCHREALTIME +echo "pass: $(awk -v a="$start" -v b="$end" 'BEGIN { printf "%.1f", b - a }') s (collect + sort + evict, rc=$rc)" +tail -n 3 "$work/prune.log" diff --git a/tests/harness.sh b/tests/harness.sh new file mode 100644 index 0000000..a25164f --- /dev/null +++ b/tests/harness.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Test harness shared by tests/*.sh. Sourced, not run. Gives a scratch dir +# ($work, removed on exit), check bookkeeping (fail / gone / kept / finish) and +# at_exit for extra cleanup. Knows nothing about what is under test; the pruner +# fixtures live in tests/lib.sh. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # tests/ +# shellcheck disable=SC2034 # used by the tests that source this +root="$(cd "$here/.." && pwd)" # repo root +work="$(mktemp -d "${TMPDIR:-/tmp}/test.XXXXXX")" + +cleanups=() +# at_exit : run it when the test ends, before $work is removed. +at_exit() { cleanups+=("$*"); } +run_cleanups() { + local c + for c in "${cleanups[@]}"; do eval "$c" || true; done + rm -rf "$work" +} +trap run_cleanups EXIT + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } + +gone() { if [ -e "$1" ] || [ -L "$1" ]; then fail "$1 should be evicted"; fi; } +kept() { [ -e "$1" ] || [ -L "$1" ] || fail "$1 should survive"; } + +# finish : end of a test; fails the run when a check failed. +finish() { + if [ "$fails" -gt 0 ]; then + echo "$fails check(s) failed" + exit 1 + fi + echo "PASS: $1" +} diff --git a/tests/image-smoke.sh b/tests/image-smoke.sh index d00ee6d..961442a 100755 --- a/tests/image-smoke.sh +++ b/tests/image-smoke.sh @@ -6,7 +6,9 @@ # (runner-slim) must carry java/node/tools but no Android SDK. # The check snippets are single-quoted on purpose: they expand inside the image. # shellcheck disable=SC2016 -set -euo pipefail + +# shellcheck source=tests/harness.sh +source "$(dirname "${BASH_SOURCE[0]}")/harness.sh" image="${1:?usage: tests/image-smoke.sh [jdk-major] [node-major] [std|ndk|slim]}" want_jdk="${2:-}" @@ -17,13 +19,16 @@ case "$variant" in *) echo "unknown variant '$variant' (want std, ndk or slim)" >&2; exit 2 ;; esac -fails=0 -fail() { echo "FAIL: $*"; fails=$((fails + 1)); } pass() { echo "ok: $*"; } indent() { printf ' %s\n' "${1//$'\n'/$'\n '}"; } +# One container, kept idle, runs every check: a docker run per check cost +# ~0.5 s of startup each. +container="$(docker run -d --rm --entrypoint sleep "$image" infinity)" +at_exit "docker rm -f $container >/dev/null 2>&1" + # in_image