From 32af9bb8ba462a8dbf44bb3bebde09288243168c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Sep 2026 09:06:25 +0000 Subject: [PATCH 01/11] Update JDK (27) versions --- matrix.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/matrix.json b/matrix.json index bb6d3f8..01630f8 100644 --- a/matrix.json +++ b/matrix.json @@ -6,7 +6,8 @@ "23", "24", "25", - "26" + "26", + "27" ], - "default_jdk": "26" + "default_jdk": "27" } From 60ac34408c74b59f2089f70f0503555b02e087da Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:57:05 +0000 Subject: [PATCH 02/11] Update myoung34/github-runner Docker tag to v2.338.0 (#85) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 102cc73..f6397fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG VERSION=2.337.0-ubuntu-noble +ARG VERSION=2.338.0-ubuntu-noble ARG JAVA_VERSION=21 ARG COMPILE_SDK=36.1 ARG BUILD_TOOLS=37.0.0 From 23e506fdf48252768a04a1b5e50c4b156ab6ee49 Mon Sep 17 00:00:00 2001 From: Jason Ernst Date: Thu, 8 Oct 2026 13:52:18 -0700 Subject: [PATCH 03/11] deploy: also tag images with the NDK version they ship Adds -jdkNN-sdkNN-ndkX.Y.Z next to the existing tag so downstream repos that pin an Android NDK (Gradle ndkVersion) can have Renovate derive it from the image tag and bump image and NDK together. The weekly SDK/NDK automation already rewrites ARG NDK_VERSION, so the tag follows it. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/deploy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1c15512..41905cc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -42,8 +42,9 @@ jobs: RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) + NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ]; then + if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$NDK_VERSION" ]; then echo "::error::Failed to extract versions from Dockerfile" exit 1 fi @@ -51,6 +52,7 @@ jobs: echo "runner_tag=${RUNNER_TAG}" >> $GITHUB_OUTPUT echo "runner_version=${RUNNER_VERSION}" >> $GITHUB_OUTPUT echo "compile_sdk=${COMPILE_SDK}" >> $GITHUB_OUTPUT + echo "ndk_version=${NDK_VERSION}" >> $GITHUB_OUTPUT - name: Login to Docker Hub uses: docker/login-action@v4 @@ -78,6 +80,9 @@ jobs: tags: | type=raw,value=jdk${{ matrix.java_version }} type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }} + # Same image, with the NDK it ships in the tag: consumers that pin an NDK (Android + # Gradle ndkVersion) can let Renovate derive it from this tag so image and NDK move together. + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }} type=raw,value=latest,enable=${{ matrix.is_default_jdk }} - name: Build and push From 9263ec73e76b056e6db5bd835b1281d718618ad4 Mon Sep 17 00:00:00 2001 From: Jason Ernst Date: Mon, 12 Oct 2026 12:10:25 -0700 Subject: [PATCH 04/11] SDK check: treat an empty as stable; tag images with build-tools Stable platforms since android-37.0 carry an empty in Google's repository manifest, so is_preview() skipped every 37.x and the image stayed on platform 36.1. Only a non-empty codename marks a preview now. deploy also publishes -jdkNN-sdkX.Y-btX.Y.Z-ndkX.Y.Z so consumers can let Renovate derive compileSdk, buildToolsVersion and ndkVersion from the image pin; the existing -sdk-ndk tag is kept. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check-sdk-updates.yml | 7 ++++--- .github/workflows/deploy.yml | 8 +++++++- README.md | 6 ++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-sdk-updates.yml b/.github/workflows/check-sdk-updates.yml index 439ddf8..fb103f1 100644 --- a/.github/workflows/check-sdk-updates.yml +++ b/.github/workflows/check-sdk-updates.yml @@ -75,8 +75,9 @@ jobs: # are listed too), which then break the unattended Docker build when # sdkmanager demands the separate Android SDK Preview license. The manifest # carries reliable preview markers, so we parse it with a real XML parser - # and skip any package that is obsolete, carries a element, or - # uses the android-sdk-preview-license. + # and skip any package that is obsolete, carries a non-empty , or + # uses the android-sdk-preview-license. Stable platforms since android-37.0 + # carry an EMPTY , so only a non-empty one marks a preview. if ! wget -q "https://dl.google.com/android/repository/repository2-3.xml" -O /tmp/android-repo.xml; then echo "Failed to download SDK repository manifest" echo "updated=false" >> $GITHUB_OUTPUT @@ -94,7 +95,7 @@ jobs: if pkg.get('obsolete') == 'true': return True for e in pkg.iter(): - if local(e.tag) == 'codename': + if local(e.tag) == 'codename' and (e.text or '').strip(): return True if local(e.tag) == 'uses-license' and e.get('ref') == 'android-sdk-preview-license': return True diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 41905cc..a07fe3b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -42,9 +42,10 @@ jobs: RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) + BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$NDK_VERSION" ]; then + if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then echo "::error::Failed to extract versions from Dockerfile" exit 1 fi @@ -52,6 +53,7 @@ jobs: echo "runner_tag=${RUNNER_TAG}" >> $GITHUB_OUTPUT echo "runner_version=${RUNNER_VERSION}" >> $GITHUB_OUTPUT echo "compile_sdk=${COMPILE_SDK}" >> $GITHUB_OUTPUT + echo "build_tools=${BUILD_TOOLS}" >> $GITHUB_OUTPUT echo "ndk_version=${NDK_VERSION}" >> $GITHUB_OUTPUT - name: Login to Docker Hub @@ -83,6 +85,10 @@ jobs: # Same image, with the NDK it ships in the tag: consumers that pin an NDK (Android # Gradle ndkVersion) can let Renovate derive it from this tag so image and NDK move together. type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }} + # Same image, with every Gradle-pinnable SDK component in the tag (compileSdk, build-tools, + # NDK), so consumers can let Renovate derive all three from one tag. The -sdk-ndk tag above + # stays for consumers already parsing it. + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-ndk${{ steps.versions.outputs.ndk_version }} type=raw,value=latest,enable=${{ matrix.is_default_jdk }} - name: Build and push diff --git a/README.md b/README.md index 995993f..ffb0d70 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,12 @@ version, and Android SDK level. Tags let you choose how tightly to pin: | `latest` | `latest` | Nothing | Runner, JDK, and SDK all track latest | | `jdk` | `jdk21` | JDK major version | Runner and SDK track latest | | `-jdk-sdk` | `2.332.0-jdk21-sdk36` | Everything | Nothing — fully pinned | +| `-jdk-sdk-ndk` | `2.338.0-jdk21-sdk36.1-ndk30.0.16248370` | Same image, NDK spelled out | Nothing — fully pinned | +| `-jdk-sdk-bt-ndk` | `2.338.0-jdk21-sdk37.2-bt37.0.0-ndk30.0.16248370` | Same image, build-tools and NDK spelled out | Nothing — fully pinned | + +The longer tags exist so a consumer can let Renovate derive its Gradle +`compileSdk`, `buildToolsVersion` and `ndkVersion` from the image pin: when they +match what the image ships, AGP doesn't download SDK components on every job. ### Which tag should I use? From 1c0fc592c7c02d655244dd47869bf115468b464d Mon Sep 17 00:00:00 2001 From: Jason Ernst Date: Mon, 12 Oct 2026 12:17:55 -0700 Subject: [PATCH 05/11] README: same-image tag examples use the same (current) SDK level Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffb0d70..8a8ac71 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ version, and Android SDK level. Tags let you choose how tightly to pin: | `jdk` | `jdk21` | JDK major version | Runner and SDK track latest | | `-jdk-sdk` | `2.332.0-jdk21-sdk36` | Everything | Nothing — fully pinned | | `-jdk-sdk-ndk` | `2.338.0-jdk21-sdk36.1-ndk30.0.16248370` | Same image, NDK spelled out | Nothing — fully pinned | -| `-jdk-sdk-bt-ndk` | `2.338.0-jdk21-sdk37.2-bt37.0.0-ndk30.0.16248370` | Same image, build-tools and NDK spelled out | Nothing — fully pinned | +| `-jdk-sdk-bt-ndk` | `2.338.0-jdk21-sdk36.1-bt37.0.0-ndk30.0.16248370` | Same image, build-tools and NDK spelled out | Nothing — fully pinned | The longer tags exist so a consumer can let Renovate derive its Gradle `compileSdk`, `buildToolsVersion` and `ndkVersion` from the image pin: when they From 42485bf3ee599b1c3dad3a3756b2aef94790b4ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Oct 2026 19:23:24 +0000 Subject: [PATCH 06/11] Update SDK versions --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f6397fb..df3aec1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=2.338.0-ubuntu-noble ARG JAVA_VERSION=21 -ARG COMPILE_SDK=36.1 +ARG COMPILE_SDK=37.2 ARG BUILD_TOOLS=37.0.0 ARG NDK_VERSION=30.0.16248370 ARG SDK_TOOLS=8512546_latest From cca70893e0ae5e2c6ecbb7b2af585a6814c4375d Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:04:16 +0300 Subject: [PATCH 07/11] refactor(pruner): one table-driven LRU engine with per-slot budgets that never prunes a busy slot - a store table + one lru_entries engine replace the three bespoke LRU functions; whole entries where partial ones break builds - bounds all 13 slot stores (was 5): bun, npm, pnpm, gradle, cargo, rustup, konan, mise, hk, pub, hosted tool cache (#7) - gradle evicts whole dependency versions, transforms and wrapper dists, never single files (#17) - host budget = min(CACHE_BUDGET_BYTES, CACHE_BUDGET_PCT of the data disk), split across slots, global LRU within a slot - skips slots marked busy (fresh marker) under the slot's .prune.lock; early pass and tighter docker prune when disk is low - tests for bun, gradle, cargo, pnpm, rustup, toolcache, busy slots and the budget split Co-Authored-By: Claude Opus 5.5 --- pruner.sh | 425 +++++++++++++++++++++++++++----------- tests/lib.sh | 67 ++++++ tests/pruner-budget.sh | 37 ++++ tests/pruner-bun.sh | 54 ++--- tests/pruner-busy.sh | 39 ++++ tests/pruner-cargo.sh | 45 ++++ tests/pruner-gradle.sh | 51 +++++ tests/pruner-pnpm.sh | 40 ++++ tests/pruner-rustup.sh | 41 ++++ tests/pruner-toolcache.sh | 30 +++ 10 files changed, 668 insertions(+), 161 deletions(-) create mode 100755 tests/lib.sh create mode 100755 tests/pruner-budget.sh create mode 100755 tests/pruner-busy.sh create mode 100755 tests/pruner-cargo.sh create mode 100755 tests/pruner-gradle.sh create mode 100755 tests/pruner-pnpm.sh create mode 100755 tests/pruner-rustup.sh create mode 100755 tests/pruner-toolcache.sh diff --git a/pruner.sh b/pruner.sh index 9dedb1b..17f0681 100755 --- a/pruner.sh +++ b/pruner.sh @@ -1,17 +1,26 @@ #!/bin/bash -# Nightly pruner. Run by the `pruner` compose service (same image, this script -# as the entrypoint), NOT by a runner job hook. +# Pruner. Run by the `pruner` compose service (same image, this script as the +# entrypoint), NOT by a runner job hook. # # These scans used to sit in cleanup.sh and ran after every job, costing # 21-39 s of runner time each time and freeing nothing (issue #16). Here they -# run once a day, off the job path: -# 1. LRU size caps on the per-runner cache volumes (mounted under -# $CACHES_ROOT//, see docker-compose.yml). Caches whose -# entries are only valid whole (bun packages, mise tool versions) evict -# whole entries; the rest evict single files. +# run off the job path: +# 1. A size budget per runner slot ($RUNNER_DATA_DIR/slots//cache/), +# enforced with one global LRU across all the stores of the slot. What is +# evictable is described by STORE_TABLE below: caches whose entries are only +# valid whole (bun packages, gradle versions, mise tool versions, ...) evict +# whole directories, the rest evict single files. # 2. docker system prune / builder prune on the host daemon (docker.sock). # 3. Hourly, in the background: fill the shared action archive cache. # +# Job safety: a slot is pruned only while holding its `.prune.lock` (pre-job.sh +# waits on it) and only when it is not mid-job: a `busy` marker younger than 6 h +# means a job is running there, so the slot is skipped; an older one is stale. +# +# Adaptive schedule: every PRUNE_CHECK_SECONDS a cheap df runs; a full pass +# happens when free space is below PRUNE_FREE_PCT, or when PRUNE_INTERVAL_SECONDS +# have passed since the last pass. Under disk pressure docker prune is tighter. +# # Action archive cache: the runner copies # $ACTION_CACHE_DIR/_/.tar.gz instead of downloading when it # exists, but never writes it. prefetch_actions() fills it as the GitHub App, so @@ -20,152 +29,303 @@ # `pruner.sh prefetch` runs it once (rollout warm-up). # # Env: -# PRUNE_INTERVAL_SECONDS sleep between runs (default 86400) -# PRUNE_INITIAL_DELAY sleep before the first run (default 300; keeps a +# RUNNER_DATA_DIR data root holding slots// (/var/lib/github-runner) +# CACHE_BUDGET_BYTES total cache budget over all slots (60 GiB) +# CACHE_BUDGET_PCT ... capped to this % of the filesystem holding +# RUNNER_DATA_DIR (40); the smaller wins and is split +# evenly between the slots +# PRUNE_CHECK_SECONDS df check period (3600) +# PRUNE_FREE_PCT full pass when free space is below (15) +# PRUNE_INTERVAL_SECONDS full pass at least this often (86400) +# PRUNE_INITIAL_DELAY sleep before the first check (300; keeps a # `docker compose up -d` from running a scan at once) -# PRUNE_RUN_ONCE=true run once and exit (manual / testing) -# CACHES_ROOT where the cache volumes are mounted (/caches) -# CACHE_CAP_BYTES cap for each file-level cache dir (20 GiB) -# MISE_DATA_CAP_BYTES cap for each mise installs tree (20 GiB) +# PRUNE_RUN_ONCE=true run one pass and exit (manual / testing) # ACTION_CACHE_CAP_BYTES cap for the action archive cache (2 GiB) # PREFETCH_SOURCES / PREFETCH_EXTRA "owner/repo@ref ..." to scan / fetch as-is # PREFETCH_INTERVAL_SECONDS prefetch period (default 3600; 0 = off) -# DOCKER_PRUNE_UNTIL / DOCKER_BUILDER_PRUNE_UNTIL (24h / 72h) +# DOCKER_PRUNE_UNTIL / DOCKER_PRUNE_UNTIL_LOW / DOCKER_BUILDER_PRUNE_UNTIL +# (24h / 2h when below PRUNE_FREE_PCT / 72h) # # Intentionally not using set -e: one failing step must not stop the others. -CACHES_ROOT="${CACHES_ROOT:-/caches}" -CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB -MISE_DATA_CAP_BYTES="${MISE_DATA_CAP_BYTES:-21474836480}" # 20 GiB +RUNNER_DATA_DIR="${RUNNER_DATA_DIR:-/var/lib/github-runner}" +CACHE_BUDGET_BYTES="${CACHE_BUDGET_BYTES:-64424509440}" # 60 GiB +CACHE_BUDGET_PCT="${CACHE_BUDGET_PCT:-40}" +PRUNE_CHECK_SECONDS="${PRUNE_CHECK_SECONDS:-3600}" +PRUNE_FREE_PCT="${PRUNE_FREE_PCT:-15}" +PRUNE_INTERVAL_SECONDS="${PRUNE_INTERVAL_SECONDS:-86400}" +BUSY_STALE_MINUTES=360 # a `busy` marker older than 6 h is a crashed job's leftover ACTION_CACHE_DIR=/opt/actionarchivecache # mount point in docker-compose.yml ACTION_CACHE_CAP_BYTES="${ACTION_CACHE_CAP_BYTES:-2147483648}" # 2 GiB PREFETCH_SOURCES="${PREFETCH_SOURCES:-}" PREFETCH_EXTRA="${PREFETCH_EXTRA:-}" GITHUB_API="https://api.github.com" DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" +DOCKER_PRUNE_UNTIL_LOW="${DOCKER_PRUNE_UNTIL_LOW:-2h}" DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } -dir_bytes() { - du -sb "$1" 2>/dev/null | awk '{print $1}' -} +# Evictable units per store, one rule per line: store|kind|glob|skip +# kind d: every directory matching glob (relative to the store dir, never a +# symlink) is one unit, removed whole; skip is a case pattern for the +# relative path of units to leave alone. Glob "." makes the whole +# store dir one unit (pnpm: store/ + cache/ are all-or-nothing); it is +# emptied, but its subdirs are kept, as container symlinks point there. +# kind f: every regular file named skip (a find -name pattern) under the dir +# glob is one unit. +# Anything no rule matches (bun's bin/ and @t@ piles, cargo's bin/ and registry +# index, rustup's settings.toml, ...) is never touched. Hooks for the few stores +# needing more than a rule are in unit_key / unit_protected / unit_pre_evict / +# unit_post_evict. Leftovers of killed bun extractions (*.tmp, *.part) are +# pre-job.sh's job: it sweeps them before the next job of a crashed slot. +STORE_TABLE=' +bun|d|install/cache/*@*|install/cache/@* +bun|d|install/cache/@*/*@*|install/cache/@t@/* +npm|f|_cacache|* +pnpm|d|.| +gradle|d|caches/modules-2/files-2.1/*/*/*| +gradle|d|caches/transforms-*/*| +gradle|d|wrapper/dists/*| +gradle|d|caches/jars-*/*| +gradle|f|caches/build-cache-1|* +cargo|f|registry/cache|*.crate +cargo|d|registry/src/*/*| +cargo|d|git/checkouts/*| +cargo|d|git/db/*| +rustup|d|toolchains/*| +konan|d|dependencies/*| +konan|d|kotlin-native-prebuilt-*| +mise-data|d|installs/*/*| +pub|d|hosted/*/*| +pub|d|git/*| +toolcache|d|*/*| +mise-cache|f|.|* +hk|f|.|* +actions|f|.|* +' -# Print how many bytes dir $1 is over cap $2. Returns 1 (printing nothing) when -# the dir is missing, unreadable, or already within the cap. -excess_bytes() { - local dir="$1" cap="$2" size - [ -d "$dir" ] || return 1 - size=$(dir_bytes "$dir") - [ -n "$size" ] || return 1 - if [ "$size" -le "$cap" ]; then - log "$dir ${size}B <= cap ${cap}B" >&2 - return 1 - fi - log "$dir is ${size}B (cap ${cap}B)" >&2 - echo $((size - cap)) +# Stores of a runner slot (everything but the shared action archive cache). +slot_stores() { + awk -F'|' 'NF > 3 && $1 != "actions" && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" } -# LRU-evict regular files, least recently accessed first, until $1 is at or -# under $2 bytes. Ordered by atime (a read counts as use; Gradle never touches -# mtime on read, so mtime order evicted the most-used dependencies first). -# Volumes mounted relatime still refresh atime on the first read after a -# change. Does not wholesale wipe the tree. -lru_cap() { - local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" - local excess freed=0 rec fsize path - excess=$(excess_bytes "$dir" "$cap") || return 0 - log "evicting least recently used files from $dir" - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - rec="${rec#* }" # drop atime - fsize="${rec%% *}" - path="${rec#* }" - if [ -z "$path" ] || [ -z "$fsize" ]; then continue; fi - rm -f "$path" 2>/dev/null || true - freed=$((freed + fsize)) - done < <(find "$dir" -type f -printf '%A@\t%s\t%p\0' 2>/dev/null | sort -z -n) - find "$dir" -type d -empty -delete 2>/dev/null || true - log "$dir freed ~${freed}B" +dir_bytes() { + du -sb "$1" 2>/dev/null | awk '{printf "%.0f\n", $1}' } -# mise installs (installs//) are only useful whole: deleting -# single files would leave a tool that exists but is broken. Evict whole -# / directories instead, oldest install first, until the -# installs tree fits under the cap. -lru_cap_mise_installs() { - local data="$1" cap="${2:-$MISE_DATA_CAP_BYTES}" - local installs="$data/installs" - local excess freed=0 rec path dsize - excess=$(excess_bytes "$installs" "$cap") || return 0 - log "evicting oldest tool versions from $installs" - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - path="${rec#* }" - [ -d "$path" ] || continue - dsize=$(dir_bytes "$path") - rm -rf "$path" 2>/dev/null || true - freed=$((freed + ${dsize:-0})) - done < <(find "$installs" -mindepth 2 -maxdepth 2 -type d -printf '%T@\t%p\0' 2>/dev/null | sort -z -n) - log "$installs freed ~${freed}B" +# default_toolchain from rustup's settings.toml ($1 = RUSTUP_HOME). +rustup_default() { + sed -n 's/^default_toolchain[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1/settings.toml" 2>/dev/null | head -n 1 } # bun treats install/cache/@@@@/ as complete whenever the folder # exists, so a file evicted from it is silently missing from every later # node_modules (next@16.3.8 lost dist/compiled/server-only/empty.js, breaking # every Next build). 0-byte files went first: copying them never reads, so -# their atime never moved. Evict whole package folders instead, least recently -# used first by the atime of their package.json (read on every install and -# import), until the whole volume ($1, mounted from /root/.bun) fits under the -# cap. Never touches $1/bin (the bun binary itself) or the @t@ manifest piles. -lru_cap_bun() { - local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" - local cache="$dir/install/cache" - local excess freed=0 rec path dsize base t - [ -d "$cache" ] || return 0 - # Leftovers of interrupted extractions; the age guard spares live installs. - find "$cache" -mindepth 1 -maxdepth 2 ! -name .tmp \( -name '*.tmp' -o -name '*.part' \) \ - -mmin +60 -exec rm -rf {} + 2>/dev/null - find "$cache/.tmp" -mindepth 1 -maxdepth 1 -mmin +60 -exec rm -rf {} + 2>/dev/null - excess=$(excess_bytes "$dir" "$cap") || return 0 - log "evicting least recently used packages from $cache" +# their atime never moved. Hence whole packages, least recently used first by +# the atime of their package.json (read on every install and import). + +# LRU key override for dir unit $2 of store $1; prints nothing to keep the +# newest-atime-of-any-file key. +unit_key() { + [ "$1" = bun ] || return 0 + stat -c %X "$2/package.json" 2>/dev/null || stat -c %Y "$2" 2>/dev/null +} + +# Succeeds when unit $3 (relative to dir $2 of store $1) must not be evicted. +unit_protected() { + local def + [ "$1" = rustup ] || return 1 + def=$(rustup_default "$2") + [ -n "$def" ] || return 1 + case "${3##*/}" in + "$def" | "$def"-*) return 0 ;; # `stable` is the dir stable- + esac + return 1 +} + +# toolcache: /.complete says / is whole. Drop the markers +# first so an interrupted removal never leaves a "complete" half tree. +unit_pre_evict() { + [ "$1" = toolcache ] || return 0 + find "$2" -mindepth 1 -maxdepth 1 -name '*.complete' -delete 2>/dev/null +} + +# After unit $2 of store $1 is gone. +unit_post_evict() { + local base + [ "$1" = bun ] || return 0 + # Drop the /@@@ symlink pointing at it, and the dir if empty. + base="${2##*/}" + rm -f "${2%/*}/${base%%@*}/${base#*@}" 2>/dev/null + rmdir "${2%/*}/${base%%@*}" 2>/dev/null + return 0 +} + +# Print NUL-terminated "keysizestorestoppath" records, one +# per evictable unit of store $1 living in dir $2. key is the LRU order (atime, +# seconds); stop is the dir above which emptied parents are not removed. +collect_units() { + local store="$1" sd="$2" s kind glob skip pre stop + [ -d "$sd" ] || return 0 + while IFS='|' read -r s kind glob skip; do + [ "$s" = "$store" ] || continue + if [ "$kind" = f ]; then + [ -d "$sd/$glob" ] || continue + find "$sd/$glob" -type f -name "$skip" -printf "%A@\t%s\t$store\t-\t%p\0" 2>/dev/null + continue + fi + pre="${glob%%[*?[]*}" + case "$pre" in + */*) pre="${pre%/*}" ;; + *) pre="" ;; + esac + stop="$sd${pre:+/$pre}" + ( + cd "$sd" 2>/dev/null || exit 0 + shopt -s nullglob + # shellcheck disable=SC2086 # glob expansion is the point + for rel in $glob; do + if [ ! -d "$rel" ] || [ -L "$rel" ]; then continue; fi + # shellcheck disable=SC2254 # skip is a pattern + case "$rel" in $skip) continue ;; esac + if unit_protected "$store" "$sd" "$rel"; then continue; fi + stats=$(find "$rel" -printf '%y\t%A@\t%s\n' 2>/dev/null | + awk -F'\t' '{ s += $3; if ($1 == "f" && $2 + 0 > m) m = $2 + 0 } END { printf "%.6f\t%.0f\n", m, s }') + key="${stats%%$'\t'*}" + t=$(unit_key "$store" "$rel") + [ -z "$t" ] || key="$t" + printf '%s\t%s\t%s\t%s\t%s\0' "$key" "${stats#*$'\t'}" "$store" "$stop" "$sd/$rel" + done + ) + done <<<"$STORE_TABLE" +} + +# Remove unit $3 of store $1 (stop dir $2): a whole directory (then its emptied +# parents up to $2), or a single file. +evict_unit() { + local store="$1" stop="$2" path="$3" p + if [ "${path##*/}" = . ]; then + find "${path%/.}" -mindepth 2 -delete 2>/dev/null + find "${path%/.}" -mindepth 1 -maxdepth 1 ! -type d -delete 2>/dev/null + elif [ -d "$path" ] && [ ! -L "$path" ]; then + unit_pre_evict "$store" "$path" + rm -rf "$path" 2>/dev/null + unit_post_evict "$store" "$path" + p="${path%/*}" + while [ "$p" != "$stop" ] && [ "${p#"$stop"/}" != "$p" ]; do + rmdir "$p" 2>/dev/null || break + p="${p%/*}" + done + else + rm -f "$path" 2>/dev/null + fi +} + +# Keep the stores given as = pairs ($2...) within $1 bytes in total, +# evicting units across all of them in global LRU order, least recently used +# first (a read counts as use; Gradle never touches mtime on read, so mtime +# order evicted the most-used dependencies first). Volumes mounted relatime +# still refresh atime on the first read after a change. +lru_entries() { + local budget="$1" spec store sd total=0 sz excess freed=0 n=0 + local rec key size stop path + local -A dirs=() touched=() + shift + for spec in "$@"; do + store="${spec%%=*}" + sd="${spec#*=}" + [ -d "$sd" ] || continue + dirs[$store]="$sd" + sz=$(dir_bytes "$sd") + total=$((total + ${sz:-0})) + done + if [ "$total" -le "$budget" ]; then + log "${total}B <= budget ${budget}B" + return 0 + fi + excess=$((total - budget)) + log "${total}B (budget ${budget}B), evicting least recently used entries" while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - path="${rec#* }" - [ -d "$path" ] || continue - dsize=$(dir_bytes "$path") - rm -rf "$path" 2>/dev/null || true - freed=$((freed + ${dsize:-0})) - # Drop the /@@@ symlink pointing at it, and the dir if empty. - base="${path##*/}" - rm -f "${path%/*}/${base%%@*}/${base#*@}" 2>/dev/null - rmdir "${path%/*}/${base%%@*}" 2>/dev/null + if [ "$freed" -ge "$excess" ]; then break; fi + IFS=$'\t' read -r key size store stop path <<<"$rec" + if [ -z "$path" ] || [ -z "$size" ]; then continue; fi + evict_unit "$store" "$stop" "$path" + freed=$((freed + size)) + n=$((n + 1)) + touched[$store]=1 done < <( - # @... at the top level, @scope/@... one level down. - find "$cache" -mindepth 1 -maxdepth 2 -type d -name '*@*' ! -name '@*' \ - \( -path "$cache/@*/*" -o ! -path "$cache/*/*" \) -print0 2>/dev/null | - while IFS= read -r -d '' path; do - t=$(stat -c %X "$path/package.json" 2>/dev/null || stat -c %Y "$path" 2>/dev/null) || continue - printf '%s\t%s\0' "$t" "$path" - done | sort -z -n + for store in "${!dirs[@]}"; do collect_units "$store" "${dirs[$store]}"; done | + LC_ALL=C sort -z -n ) - find "$cache" -mindepth 1 -maxdepth 1 -name '@*' ! -name '@t@' -type d -empty -delete 2>/dev/null || true - log "$cache freed ~${freed}B" + # Emptied directories under file-level stores. + for store in "${!touched[@]}"; do + while IFS='|' read -r s kind glob _; do + if [ "$s" = "$store" ] && [ "$kind" = f ] && [ -d "${dirs[$store]}/$glob" ]; then + find "${dirs[$store]}/$glob" -mindepth 1 -type d -empty -delete 2>/dev/null + fi + done <<<"$STORE_TABLE" + done + if [ "$freed" -lt "$excess" ]; then + log "evicted $n entries, ~${freed}B freed; still ~$((excess - freed))B over budget (protected or untracked data)" + else + log "evicted $n entries, ~${freed}B freed" + fi +} + +# Total cache budget in bytes of one slot, with $1 slots sharing it. +slot_budget() { + local slots="$1" fs total + total="$CACHE_BUDGET_BYTES" + fs=$(df -B1 --output=size "$RUNNER_DATA_DIR" 2>/dev/null | awk 'NR == 2 {print $1}') + if [ -n "$fs" ] && [ $((fs * CACHE_BUDGET_PCT / 100)) -lt "$total" ]; then + total=$((fs * CACHE_BUDGET_PCT / 100)) + fi + echo $((total / slots)) +} + +# Free space of the filesystem holding RUNNER_DATA_DIR, in whole percent. +free_pct() { + local dir="$RUNNER_DATA_DIR" + [ -d "$dir" ] || dir=/ + df -B1 --output=avail,size "$dir" 2>/dev/null | awk 'NR == 2 && $2 > 0 {print int($1 * 100 / $2)}' +} + +# Prune one slot ($1 = slot dir) down to $2 bytes, unless a job is running in +# it. The flock keeps jobs from starting meanwhile: pre-job.sh waits on it. +prune_slot() { + local slot="$1" budget="$2" id start="$SECONDS" + id="${slot##*/}" + ( + flock -n 9 || { log "slot $id: another prune holds the lock; skipping"; exit 0; } + if [ -e "$slot/busy" ]; then + if [ -n "$(find "$slot/busy" -maxdepth 0 -mmin "-$BUSY_STALE_MINUTES" 2>/dev/null)" ]; then + log "slot $id: job running (busy marker); skipping" + exit 0 + fi + log "slot $id: stale busy marker (> ${BUSY_STALE_MINUTES} min); pruning anyway" + fi + args=() + for store in $(slot_stores); do args+=("$store=$slot/cache/$store"); done + log "slot $id: budget ${budget}B" + lru_entries "$budget" "${args[@]}" + ) 9>"$slot/.prune.lock" + log "slot $id: took $((SECONDS - start))s" } prune_caches() { - local runner - for runner in "$CACHES_ROOT"/*/; do - [ -d "$runner" ] || continue - runner="${runner%/}" - log "capping caches under $runner" - lru_cap "$runner/gradle" - lru_cap_bun "$runner/bun" - lru_cap "$runner/mise-cache" - lru_cap "$runner/hk" # configs/.json + pklr/packages/.zip: whole files - lru_cap_mise_installs "$runner/mise-data" + local caches=() cache budget + for cache in "$RUNNER_DATA_DIR"/slots/*/cache; do + [ -d "$cache" ] && caches+=("$cache") done - lru_cap "$ACTION_CACHE_DIR" "$ACTION_CACHE_CAP_BYTES" + if [ "${#caches[@]}" -gt 0 ]; then + budget=$(slot_budget "${#caches[@]}") + for cache in "${caches[@]}"; do prune_slot "${cache%/cache}" "$budget"; done + else + log "no slots under $RUNNER_DATA_DIR/slots" + fi + lru_entries "$ACTION_CACHE_CAP_BYTES" "actions=$ACTION_CACHE_DIR" } gh_curl() { @@ -199,7 +359,8 @@ prefetch_one() { fi file="$dir/$sha.tar.gz" [ -s "$file" ] && return 0 - mkdir -p "$dir" && tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 + mkdir -p "$dir" || return 1 + tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 if gh_curl -o "$tmp" "$GITHUB_API/repos/$name/tarball/$sha" && tar -tzf "$tmp" >/dev/null 2>&1; then chmod 644 "$tmp" && mv -f "$tmp" "$file" && { log "prefetch: cached $name@$sha ($ref)" >&2; echo 1; return 0; } fi @@ -253,13 +414,16 @@ prefetch_actions() { GH_TOKEN="" } +# $1 = low: free space is below PRUNE_FREE_PCT, so prune docker harder. prune_docker() { + local until="$DOCKER_PRUNE_UNTIL" + [ "${1:-}" = low ] && until="$DOCKER_PRUNE_UNTIL_LOW" if ! docker info >/dev/null 2>&1; then log "docker daemon unreachable; skipping docker prune" return 0 fi - log "docker system prune -af --filter until=${DOCKER_PRUNE_UNTIL}" - docker system prune -af --filter "until=${DOCKER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' + log "docker system prune -af --filter until=${until}" + docker system prune -af --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' log "docker builder prune -af --filter until=${DOCKER_BUILDER_PRUNE_UNTIL}" docker builder prune -af --filter "until=${DOCKER_BUILDER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' } @@ -268,7 +432,7 @@ run_once() { local before after before=$(df -h / | awk 'NR==2 {print $4}') prune_caches - prune_docker + prune_docker "${1:-}" after=$(df -h / | awk 'NR==2 {print $4}') log "done. Disk free: ${before} -> ${after}" } @@ -297,7 +461,20 @@ if [ "${PREFETCH_INTERVAL_SECONDS:-3600}" -gt 0 ]; then fi sleep "${PRUNE_INITIAL_DELAY:-300}" +last=0 while true; do - run_once - sleep "${PRUNE_INTERVAL_SECONDS:-86400}" + free=$(free_pct) + low="" + now=$(date +%s) + if [ -n "$free" ] && [ "$free" -lt "$PRUNE_FREE_PCT" ]; then + low=low + log "free space ${free}% < ${PRUNE_FREE_PCT}%: full pass" + elif [ $((now - last)) -ge "$PRUNE_INTERVAL_SECONDS" ]; then + log "free space ${free:-?}%, ${PRUNE_INTERVAL_SECONDS}s since the last pass: full pass" + fi + if [ -n "$low" ] || [ $((now - last)) -ge "$PRUNE_INTERVAL_SECONDS" ]; then + run_once "$low" + last=$(date +%s) + fi + sleep "$PRUNE_CHECK_SECONDS" done diff --git a/tests/lib.sh b/tests/lib.sh new file mode 100755 index 0000000..9538064 --- /dev/null +++ b/tests/lib.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Shared helpers for tests/pruner-*.sh. Sourced, not run. Needs GNU coreutils, +# find, flock (Linux, or `docker run --rm -v "$PWD:/w" -w /w ubuntu:24.04 bash tests/.sh`). +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } + +# shellcheck source=pruner.sh +source "$here/../pruner.sh" +RUNNER_DATA_DIR="$work/data" +ACTION_CACHE_DIR="$work/no-action-cache" +CACHE_BUDGET_PCT=100 # the byte budget decides, whatever the test fs is + +# mkslot : create slot and print its cache dir. +mkslot() { + mkdir -p "$RUNNER_DATA_DIR/slots/$1/cache" "$RUNNER_DATA_DIR/slots/$1/work" + touch "$RUNNER_DATA_DIR/slots/$1/.lock" + echo "$RUNNER_DATA_DIR/slots/$1/cache" +} + +sz() { du -sb "$1" | awk '{printf "%.0f\n", $1}'; } + +# stores_bytes : what the pruner counts for a slot. +stores_bytes() { + local d total=0 + for d in "$1"/*/; do total=$((total + $(sz "$d"))); done + echo "$total" +} + +# mkfile +mkfile() { + mkdir -p "$(dirname "$1")" + head -c "$2" /dev/urandom >"$1" + touch -a -d "$3 days ago" "$1" +} + +# mkdirfile : a unit with one file of that age. +mkunit() { + mkfile "$1/data.bin" "$2" "$3" +} + +# budget_without ...: a budget that makes exactly s +# (the oldest ones) go and nothing more: now minus them, plus slack that is far +# below any unit's size. +budget_without() { + local cache="$1" total u + shift + total=$(stores_bytes "$cache") + for u in "$@"; do total=$((total - $(sz "$u"))); done + echo $((total + 20000)) +} + +gone() { if [ -e "$1" ] || [ -L "$1" ]; then fail "$1 should be evicted"; fi; } +kept() { [ -e "$1" ] || [ -L "$1" ] || fail "$1 should survive"; } + +finish() { + if [ "$fails" -gt 0 ]; then + echo "$fails check(s) failed" + exit 1 + fi + echo "PASS: $1" +} diff --git a/tests/pruner-budget.sh b/tests/pruner-budget.sh new file mode 100755 index 0000000..e4aaa1f --- /dev/null +++ b/tests/pruner-budget.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Budget: min(CACHE_BUDGET_BYTES, CACHE_BUDGET_PCT% of the filesystem) split +# evenly over the slots, each slot ranked on its own (not one LRU across slots). +# Needs GNU tools: +# bash tests/pruner-budget.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +# slot_budget arithmetic. +mkdir -p "$RUNNER_DATA_DIR" +CACHE_BUDGET_BYTES=1000 CACHE_BUDGET_PCT=100 +[ "$(slot_budget 2)" = 500 ] || fail "1000B over 2 slots: $(slot_budget 2)" +fs=$(df -B1 --output=size "$RUNNER_DATA_DIR" | awk 'NR == 2 {print $1}') +CACHE_BUDGET_BYTES=$((fs * 2)) CACHE_BUDGET_PCT=10 +[ "$(slot_budget 4)" = $((fs * 10 / 100 / 4)) ] || fail "10% of the fs over 4 slots: $(slot_budget 4)" + +# Two slots, three 100000 B units each. Slot 2's units are all newer than +# slot 1's: a global LRU would empty slot 1 first; per slot, each loses just +# its own oldest unit. +for id in 1 2; do + c="$(mkslot "$id")" + if [ "$id" = 1 ]; then ages=(90 80 70); else ages=(30 20 10); fi + for i in 0 1 2; do mkunit "$c/mise-data/installs/tool/v$i" 100000 "${ages[$i]}"; done +done +per_slot=$(budget_without "$RUNNER_DATA_DIR/slots/1/cache" "$RUNNER_DATA_DIR/slots/1/cache/mise-data/installs/tool/v0") +CACHE_BUDGET_BYTES=$((per_slot * 2)) CACHE_BUDGET_PCT=100 +out=$(prune_caches) +for id in 1 2; do + u="$RUNNER_DATA_DIR/slots/$id/cache/mise-data/installs/tool" + gone "$u/v0" + kept "$u/v1/data.bin" + kept "$u/v2/data.bin" + grep -q "slot $id: budget ${per_slot}B" <<<"$out" || fail "slot $id budget not ${per_slot}B" +done + +finish "budget split evenly across slots" diff --git a/tests/pruner-bun.sh b/tests/pruner-bun.sh index aca92af..4794f7e 100755 --- a/tests/pruner-bun.sh +++ b/tests/pruner-bun.sh @@ -1,21 +1,17 @@ #!/bin/bash # pruner.sh must evict whole bun packages, never single files: bun reuses a -# cache folder as complete whenever it exists. Builds a fake /root/.bun volume -# (packages holding 0-byte files with old atimes, a scoped package, the -# / symlinks, @t@ piles and bin/bun), runs prune_caches under a small -# cap, then checks every surviving package is complete and every evicted one -# is fully gone. Needs GNU find/stat/du (Linux, or `docker run ubuntu`): +# cache folder as complete whenever it exists. Builds a fake /root/.bun store +# in a slot (packages holding 0-byte files with old atimes, a scoped package, +# the / symlinks, @t@ piles and bin/bun), runs prune_caches under a +# small budget, then checks every surviving package is complete and every +# evicted one is fully gone. Needs GNU tools (Linux, or `docker run ubuntu`): # bash tests/pruner-bun.sh -set -euo pipefail - here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -work="$(mktemp -d)" -trap 'rm -rf "$work"' EXIT - -fails=0 -fail() { echo "FAIL: $*"; fails=$((fails + 1)); } +# shellcheck source=tests/lib.sh +source "$here/lib.sh" -bun="$work/caches/1/bun" +cache0="$(mkslot 1)" +bun="$cache0/bun" cache="$bun/install/cache" mkdir -p "$bun/bin" "$cache/@t@" head -c 300000 /dev/zero >"$bun/bin/bun" @@ -50,20 +46,11 @@ rm "$cache/nopkgjson@1.0.0@@@1/package.json" # falls back to dir mtime sed -i '/^package.json$/d' "$work/nopkgjson@1.0.0@@@1.files" touch -d '1 hour ago' "$cache/nopkgjson@1.0.0@@@1" -# Stale and live partial downloads. -mkdir -p "$cache/.tmp/stale" "$cache/.tmp/live" -touch -d '2 days ago' "$cache/.tmp/stale" -head -c 10 /dev/zero >"$cache/half@1.0.0@@@1.tgz.part" -touch -d '2 days ago' "$cache/half@1.0.0@@@1.tgz.part" - -# Cap = volume minus the three oldest packages (old-a, old-b, @scope/old-c), -# plus half a package of slack: exactly those three must go, nothing else. -sz() { du -sb "$1" | awk '{print $1}'; } -cap=$(($(sz "$bun") - $(sz "$cache/old-a@1.0.0@@@1") - $(sz "$cache/old-b@2.0.0@@@1") \ - - $(sz "$cache/@scope/old-c@3.0.0@@@1") - 2 * 4096 + 50000)) -# shellcheck source=pruner.sh -source "$here/../pruner.sh" -CACHES_ROOT="$work/caches" CACHE_CAP_BYTES="$cap" prune_caches +# Budget = volume minus the three oldest packages (old-a, old-b, @scope/old-c): +# exactly those three must go, nothing else. +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$cache/old-a@1.0.0@@@1" "$cache/old-b@2.0.0@@@1" \ + "$cache/@scope/old-c@3.0.0@@@1") +prune_caches >/dev/null for entry in 'old-a@1.0.0@@@1' 'old-b@2.0.0@@@1' '@scope/old-c@3.0.0@@@1'; do base="${entry##*/}" @@ -89,15 +76,8 @@ done [ -L "$bun/bin/bunx" ] || fail "bin/bunx removed" [ -f "$cache/@t@/0061322e0ad01aa4.pile" ] || fail "@t@ pile removed" [ -d "$cache/@scope" ] || fail "@scope dir removed while it still holds a package" -[ ! -e "$cache/.tmp/stale" ] || fail "stale .tmp entry kept" -[ -e "$cache/.tmp/live" ] || fail "live .tmp entry removed" -[ ! -e "$cache/half@1.0.0@@@1.tgz.part" ] || fail "stale .part kept" -size=$(sz "$bun") -[ "$size" -le "$cap" ] || fail "volume ${size}B still over the ${cap}B cap" +size=$(stores_bytes "$cache0") +[ "$size" -le "$CACHE_BUDGET_BYTES" ] || fail "slot ${size}B still over the ${CACHE_BUDGET_BYTES}B budget" -if [ "$fails" -gt 0 ]; then - echo "$fails check(s) failed" - exit 1 -fi -echo "PASS: bun cache evicted by whole package (volume ${size}B)" +finish "bun cache evicted by whole package (slot ${size}B)" diff --git a/tests/pruner-busy.sh b/tests/pruner-busy.sh new file mode 100755 index 0000000..d5951f6 --- /dev/null +++ b/tests/pruner-busy.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Job safety: a slot with a fresh `busy` marker is skipped, one with a stale +# marker (> 6 h) is pruned, and a slot whose .prune.lock is held is skipped. +# Needs GNU tools and flock: +# bash tests/pruner-busy.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +unit() { echo "$RUNNER_DATA_DIR/slots/$1/cache/mise-data/installs/node/20.0.0"; } +for id in 1 2 3 4; do + mkslot "$id" >/dev/null + mkunit "$(unit "$id")" 100000 30 +done +touch "$RUNNER_DATA_DIR/slots/1/busy" # job running +touch -d '7 hours ago' "$RUNNER_DATA_DIR/slots/2/busy" # crashed job's leftover +# slot 3: idle +mkdir -p "$RUNNER_DATA_DIR/slots/c1" && touch "$RUNNER_DATA_DIR/slots/c1/.lock" # control slot, no cache/ +( flock 8; sleep 60 ) 8>"$RUNNER_DATA_DIR/slots/4/.prune.lock" & # another pruner +holder=$! +sleep 1 + +CACHE_BUDGET_BYTES=1 +out=$(prune_caches) +kill "$holder" 2>/dev/null || true +wait "$holder" 2>/dev/null || true + +kept "$(unit 1)/data.bin" +gone "$(unit 2)" +gone "$(unit 3)" +kept "$(unit 4)/data.bin" +[ ! -e "$RUNNER_DATA_DIR/slots/c1/cache" ] || fail "control slot got a cache dir" +grep -q 'slot 1: job running' <<<"$out" || fail "no log line for the busy slot" +grep -q 'slot 2: stale busy' <<<"$out" || fail "no log line for the stale busy marker" +grep -q 'slot 4: another prune' <<<"$out" || fail "no log line for the locked slot" +grep -q 'slot 3: took [0-9]*s' <<<"$out" || fail "no elapsed log line" +[ -e "$RUNNER_DATA_DIR/slots/1/busy" ] || fail "busy marker removed by the pruner" + +finish "busy slot skipped, stale busy pruned, locked slot skipped" diff --git a/tests/pruner-cargo.sh b/tests/pruner-cargo.sh new file mode 100755 index 0000000..c0ff9ef --- /dev/null +++ b/tests/pruner-cargo.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Cargo: .crate files, extracted crates and git checkouts/dbs are evicted; +# bin/ (cargo install'ed tools) and the registry index never are. Needs GNU tools: +# bash tests/pruner-cargo.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +cache0="$(mkslot 1)" +c="$cache0/cargo" + +mkfile "$c/bin/cargo-nextest" 200000 900 # oldest thing in the store +mkfile "$c/registry/index/idx/.cache/ab/serde" 50000 900 +mkfile "$c/registry/cache/idx/old-1.0.0.crate" 100000 90 +mkfile "$c/registry/cache/idx/fresh-1.0.0.crate" 100000 1 +mkunit "$c/registry/src/idx/old-1.0.0" 100000 80 +mkunit "$c/registry/src/idx/fresh-1.0.0" 100000 1 +mkunit "$c/git/checkouts/oldrepo-aaa" 100000 70 +mkunit "$c/git/checkouts/freshrepo-bbb" 100000 1 +mkunit "$c/git/db/oldrepo-aaa" 100000 60 +mkunit "$c/git/db/freshrepo-bbb" 100000 1 + +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$c/registry/cache/idx/old-1.0.0.crate" "$c/registry/src/idx/old-1.0.0" \ + "$c/git/checkouts/oldrepo-aaa" "$c/git/db/oldrepo-aaa") +prune_caches >/dev/null + +gone "$c/registry/cache/idx/old-1.0.0.crate" +gone "$c/registry/src/idx/old-1.0.0" +gone "$c/git/checkouts/oldrepo-aaa" +gone "$c/git/db/oldrepo-aaa" +kept "$c/registry/cache/idx/fresh-1.0.0.crate" +kept "$c/registry/src/idx/fresh-1.0.0/data.bin" +kept "$c/git/checkouts/freshrepo-bbb/data.bin" +kept "$c/git/db/freshrepo-bbb/data.bin" +[ -s "$c/bin/cargo-nextest" ] || fail "bin/cargo-nextest removed" +[ -s "$c/registry/index/idx/.cache/ab/serde" ] || fail "registry index removed" + +# Even with no budget at all, bin/ and the index stay. +CACHE_BUDGET_BYTES=1 +prune_caches >/dev/null +[ -s "$c/bin/cargo-nextest" ] || fail "bin/cargo-nextest removed under a tiny budget" +[ -s "$c/registry/index/idx/.cache/ab/serde" ] || fail "registry index removed under a tiny budget" +[ ! -e "$c/registry/cache/idx/fresh-1.0.0.crate" ] || fail "fresh crate should go under a tiny budget" + +finish "cargo evicted caches, bin/ protected" diff --git a/tests/pruner-gradle.sh b/tests/pruner-gradle.sh new file mode 100755 index 0000000..e92f8ae --- /dev/null +++ b/tests/pruner-gradle.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Gradle: whole module versions / transform dirs / wrapper dists / jars dirs +# are evicted, never single files inside them, and the metadata Gradle keeps +# beside them (modules-2/metadata-*) is left alone. Needs GNU tools: +# bash tests/pruner-gradle.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +cache0="$(mkslot 1)" +g="$cache0/gradle" +fs="$g/caches/modules-2/files-2.1" + +mkunit "$fs/org.old/lib/1.0.0/h1" 100000 90 # oldest version of a module +mkfile "$fs/org.old/lib/1.0.0/h1/lib-1.0.0.pom" 50000 400 +mkunit "$fs/org.old/lib/1.1.0/h2" 100000 2 # newer version of the same module +mkunit "$fs/org.fresh/app/2.0.0/h3" 100000 1 +mkunit "$g/caches/transforms-4/oldhash" 100000 80 +mkunit "$g/caches/transforms-4/freshhash" 100000 1 +mkunit "$g/wrapper/dists/gradle-7.0-bin/abc" 100000 70 +mkunit "$g/wrapper/dists/gradle-8.5-bin/def" 100000 1 +mkunit "$g/caches/jars-9/oldjars" 100000 60 +mkfile "$g/caches/build-cache-1/oldentry" 100000 50 +mkfile "$g/caches/build-cache-1/freshentry" 100000 1 +mkdir -p "$g/caches/modules-2/metadata-2.107" +echo meta >"$g/caches/modules-2/metadata-2.107/descriptors.bin" +touch -a -d '500 days ago' "$g/caches/modules-2/metadata-2.107/descriptors.bin" + +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$fs/org.old/lib/1.0.0" "$g/caches/transforms-4/oldhash" \ + "$g/wrapper/dists/gradle-7.0-bin" "$g/caches/jars-9/oldjars" "$g/caches/build-cache-1/oldentry") +prune_caches >/dev/null + +gone "$fs/org.old/lib/1.0.0" +gone "$g/caches/transforms-4/oldhash" +gone "$g/wrapper/dists/gradle-7.0-bin" +gone "$g/caches/jars-9/oldjars" +gone "$g/caches/build-cache-1/oldentry" +kept "$fs/org.old/lib/1.1.0/h2/data.bin" +kept "$fs/org.fresh/app/2.0.0/h3/data.bin" +kept "$g/caches/transforms-4/freshhash/data.bin" +kept "$g/wrapper/dists/gradle-8.5-bin/def/data.bin" +kept "$g/caches/build-cache-1/freshentry" +kept "$g/caches/modules-2/metadata-2.107/descriptors.bin" +[ -d "$fs/org.old/lib" ] || fail "module dir removed while a version is left" +[ -d "$fs" ] || fail "files-2.1 removed" +[ -d "$g/caches/build-cache-1" ] || fail "build-cache-1 removed" + +size=$(stores_bytes "$cache0") +[ "$size" -le "$CACHE_BUDGET_BYTES" ] || fail "slot ${size}B still over the ${CACHE_BUDGET_BYTES}B budget" + +finish "gradle evicted whole versions, metadata untouched" diff --git a/tests/pruner-pnpm.sh b/tests/pruner-pnpm.sh new file mode 100755 index 0000000..c42ddc1 --- /dev/null +++ b/tests/pruner-pnpm.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# pnpm: the store is all-or-nothing, ranked by its newest atime. Never a file +# or two out of a store (pnpm trusts the files its index lists). Needs GNU tools: +# bash tests/pruner-pnpm.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +# mkstore : a store with one very old +# file and one file of the given age. +mkstore() { + mkfile "$1/pnpm/store/v10/files/aa/old-file" 100000 300 + mkfile "$1/pnpm/cache/metadata/newest-file" 100000 "$2" +} + +# Slot 1: the pnpm dir (newest atime 10 d) is older than a gradle unit (1 d): +# all of it (store/ and cache/) must go, even though dropping its 300 d file would do. +c1="$(mkslot 1)" +mkstore "$c1" 10 +mkunit "$c1/gradle/caches/transforms-4/h" 100000 1 +CACHE_BUDGET_BYTES=$(($(stores_bytes "$c1") - 150000)) +prune_caches >/dev/null +gone "$c1/pnpm/store/v10" +gone "$c1/pnpm/cache/metadata" +if [ ! -d "$c1/pnpm/store" ] || [ ! -d "$c1/pnpm/cache" ]; then fail "pnpm store/ and cache/ dirs must stay (symlink targets)"; fi +kept "$c1/gradle/caches/transforms-4/h/data.bin" + +# Slot 2: the store is the most recently used, so the gradle unit goes and the +# store stays untouched, including its 300 d file. +rm -rf "$RUNNER_DATA_DIR/slots/1" +c2="$(mkslot 2)" +mkstore "$c2" 1 +mkunit "$c2/gradle/caches/transforms-4/h" 100000 10 +CACHE_BUDGET_BYTES=$(($(stores_bytes "$c2") - 50000)) +prune_caches >/dev/null +gone "$c2/gradle/caches/transforms-4/h" +kept "$c2/pnpm/store/v10/files/aa/old-file" +kept "$c2/pnpm/cache/metadata/newest-file" + +finish "pnpm store evicted whole or not at all" diff --git a/tests/pruner-rustup.sh b/tests/pruner-rustup.sh new file mode 100755 index 0000000..d51569d --- /dev/null +++ b/tests/pruner-rustup.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# rustup: whole toolchains are evicted, except the default one named in +# settings.toml (which may be given by its short name). Needs GNU tools: +# bash tests/pruner-rustup.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +# mkrustup +mkrustup() { + local r="$1/rustup" + mkdir -p "$r" + printf 'version = "12"\ndefault_toolchain = "%s"\nprofile = "minimal"\n' "$2" >"$r/settings.toml" + mkunit "$r/toolchains/stable-x86_64-unknown-linux-gnu" 200000 400 # oldest, but the default + mkunit "$r/toolchains/1.70.0-x86_64-unknown-linux-gnu" 100000 90 + mkunit "$r/toolchains/nightly-2024-01-01-x86_64-unknown-linux-gnu" 100000 80 + mkunit "$r/toolchains/1.80.0-x86_64-unknown-linux-gnu" 100000 1 +} + +for default in stable-x86_64-unknown-linux-gnu stable; do + rm -rf "$RUNNER_DATA_DIR/slots" + c="$(mkslot 1)" + mkrustup "$c" "$default" + t="$c/rustup/toolchains" + CACHE_BUDGET_BYTES=$(budget_without "$c" "$t/1.70.0-x86_64-unknown-linux-gnu" \ + "$t/nightly-2024-01-01-x86_64-unknown-linux-gnu") + prune_caches >/dev/null + gone "$t/1.70.0-x86_64-unknown-linux-gnu" + gone "$t/nightly-2024-01-01-x86_64-unknown-linux-gnu" + kept "$t/1.80.0-x86_64-unknown-linux-gnu/data.bin" + kept "$t/stable-x86_64-unknown-linux-gnu/data.bin" + + # No budget at all: everything but the default goes. + CACHE_BUDGET_BYTES=1 + prune_caches >/dev/null + gone "$t/1.80.0-x86_64-unknown-linux-gnu" + kept "$t/stable-x86_64-unknown-linux-gnu/data.bin" + kept "$c/rustup/settings.toml" +done + +finish "rustup default toolchain protected" diff --git a/tests/pruner-toolcache.sh b/tests/pruner-toolcache.sh new file mode 100755 index 0000000..39aa561 --- /dev/null +++ b/tests/pruner-toolcache.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# toolcache: / is evicted whole, taking its .complete marker +# with it so setup-* actions never see a "complete" tool that is gone. Needs GNU tools: +# bash tests/pruner-toolcache.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +cache0="$(mkslot 1)" +t="$cache0/toolcache" + +# mktool +mktool() { + mkfile "$t/$1/$2/x64/bin/$1" 100000 "$3" + touch -a -d "$3 days ago" "$t/$1/$2/x64.complete" +} +mktool node 18.0.0 90 +mktool node 20.0.0 1 +mktool Python 3.9.0 80 # the only Python version + +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$t/node/18.0.0" "$t/Python/3.9.0") +prune_caches >/dev/null + +gone "$t/node/18.0.0" +gone "$t/node/18.0.0/x64.complete" +gone "$t/Python" +kept "$t/node/20.0.0/x64/bin/node" +kept "$t/node/20.0.0/x64.complete" + +finish "toolcache version evicted with its .complete marker" From d8683c6bd788d6065df22817785178b4b82d6f12 Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:04:16 +0300 Subject: [PATCH 08/11] feat(compose): scalable runner slots on one host data dir, identical-path work dirs, per-host sizing - one runner service scaled by RUNNERS (+ CONTROL_RUNNERS) replaces 3 hand-copied services and 36 named volumes - each container flock-claims RUNNER_DATA_DIR/slots/ for its lifetime; work dir and caches live in the slot - the data dir is mounted at the same host path, so docker run -v, container actions and container: jobs see the checkout (#19) - default cache paths (bun, npm, pnpm, gradle, cargo, rustup, konan, mise, hk, pub, hostedtoolcache) link into the slot (#8) - pre-job marks the slot busy and sweeps leftovers only after a crashed job; cleanup clears busy and wipes /tmp flat - optional RUNNER_CPUS / RUNNER_MEMORY; under a CPU quota gradle/cargo/make/cmake workers are capped to it - hosts/sizing.sh suggests RUNNERS, limits, the pruner budget and :slim on arm64; RUNNER_CACHES=off replaces no-caches overlay - tests/slots.sh: concurrent claims, labels, links, parallelism, hooks and a store-list drift guard Co-Authored-By: Claude Opus 5.5 --- .env.example | 23 +++- cleanup.sh | 104 +++++--------- docker-compose.yml | 262 ++++++++++++------------------------ hosts/no-caches.compose.yml | 24 ---- hosts/sizing.sh | 71 ++++++++++ pre-job.sh | 47 +++++-- preflight-entrypoint.sh | 143 +++++++++++++++++++- tests/slots.sh | 210 +++++++++++++++++++++++++++++ 8 files changed, 590 insertions(+), 294 deletions(-) delete mode 100644 hosts/no-caches.compose.yml create mode 100755 hosts/sizing.sh create mode 100644 tests/slots.sh diff --git a/.env.example b/.env.example index c3d0116..03d57fc 100644 --- a/.env.example +++ b/.env.example @@ -6,22 +6,31 @@ APP_PRIVATE_KEY= # Organization the runners register to APP_LOGIN= +# How many runners this host runs; `hosts/sizing.sh [light|large]` suggests a +# value from its CPUs, memory and disk. +RUNNERS=3 +# Host directory for every runner's work dir and caches, mounted at the same +# path in the containers (slots//work, slots//cache). Put it on a disk +# with room; old named volumes from before slots can be removed. +# RUNNER_DATA_DIR=/var/lib/github-runner + # Runner identity. The runner adds self-hosted, the OS and the architecture to # LABELS itself; add the capability labels your workflows select on. RUNNER_NAME_PREFIX=runner LABELS=docker RUNNER_GROUP=Default # One runner can join an extra pool, e.g. a host whose runners are `light` with -# one that also builds: +# one that also builds (slot 3): # RUNNER_3_LABELS=docker,x86,light,large -# Coordination runners (two, labelled `control`), for the short jobs a merge -# waits on. Off unless the profile is set: -# COMPOSE_PROFILES=control +# Coordination runners labelled CONTROL_LABELS, for the short jobs a merge +# waits on. None unless set: +# CONTROL_RUNNERS=2 # CONTROL_LABELS=control -# Small hosts without room for per-runner caches (e.g. ARM64 boards): -# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml +# Small hosts without room for caches (e.g. ARM64 boards): jobs then start +# cold. The pruner's budget already shrinks with the disk, so try that first. +# RUNNER_CACHES=off # Action archive cache (see README). Space-separated owner/repo@ref lists. # PREFETCH_SOURCES=my-org/my-workflows@main # repos scanned for `uses:` @@ -29,7 +38,7 @@ RUNNER_GROUP=Default # Other tunables (defaults shown) # RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner -# CACHE_CAP_BYTES=21474836480 +# CACHE_BUDGET_BYTES=64424509440 # host-wide cache budget, split across slots # PRUNE_INTERVAL_SECONDS=86400 # PREFETCH_INTERVAL_SECONDS=3600 # Any non-empty value disables runner self-update ("false" too). diff --git a/cleanup.sh b/cleanup.sh index 29dfb19..d77b8e1 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -2,88 +2,48 @@ # Post-job cleanup script for GitHub Actions runner. # Invoked automatically via ACTIONS_RUNNER_HOOK_JOB_COMPLETED after each job. # -# Persist package, Gradle/Cargo and mise/hk caches on the named volumes. This -# hook only does the CHEAP tidy-up that has to happen after every job. The -# expensive size-cap scans (du over multi-GB trees, LRU eviction) used to run -# here and cost 21-39 s per job (issue #16); they now live in pruner.sh, run -# nightly by the `pruner` compose service. JOB_STARTED (pre-job.sh) is the -# poison sweep for a killed runner process. +# Only the CHEAP tidy-up that has to happen after every job. The size caps +# live in pruner.sh, run by the `pruner` compose service (issue #16: they used +# to cost 21-39 s per job here). JOB_STARTED (pre-job.sh) sweeps up after a +# job whose runner was killed before this hook ran. # # Intentionally not using set -e: individual cleanup failures should not # prevent other cleanups from running. # -# KEEP: $HOME/.bun, $HOME/.npm, $HOME/.cache/pnpm, $HOME/.cache/yarn, -# $HOME/.cache/mise, $HOME/.local/share/mise, $HOME/.cache/hk, -# $HOME/.konan, $HOME/.rustup, -# Gradle modules-2 / build-cache-* / transforms-*, Cargo registry. -# DELETE: /tmp contents except $RUNNER_WORKDIR (and $RUNNER_TEMP if under -# /tmp), Gradle daemon/journals/locks, runner _diag logs. +# KEEP: the slot caches (see preflight-entrypoint.sh) and the work dir, which +# lives in the slot (RUNNER_DATA_DIR/slots//work), not under /tmp. +# DELETE: /tmp contents, Gradle daemon/journals/locks, runner _diag logs. -echo "[cleanup] Running post-job cleanup..." -BEFORE=$(df -h / | awk 'NR==2 {print $4}') - -# True when $1 is $2 or lives inside $2. -_inside() { - [ -n "$2" ] || return 1 - case "$1" in "$2"|"$2"/*) return 0 ;; esac - return 1 -} - -# True when $1 is a strict ancestor of $2. -_ancestor_of() { - [ -n "$2" ] || return 1 - case "$2" in "$1"/*) return 0 ;; esac - return 1 -} - -# RUNNER_WORKDIR is /tmp/runner/work in docker-compose.yml. A blanket -# `find /tmp -delete` wipes the work volume. Spare the workdir (and -# RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp. -# -# Walk top-down and never descend into a spared tree: the work volume holds -# checkouts plus node_modules, and `find -depth` (which disables -prune) used -# to walk every path in it on every job. Only strict ancestors of a spared -# path are entered, so the cost is the handful of entries outside the workdir. -clean_dir() { - local dir="$1" workdir="$2" temp="$3" path - while IFS= read -r -d '' path; do - if _inside "$path" "$workdir" || _inside "$path" "$temp"; then - continue - fi - if _ancestor_of "$path" "$workdir" || _ancestor_of "$path" "$temp"; then - clean_dir "$path" "$workdir" "$temp" - continue - fi - rm -rf "$path" 2>/dev/null || true - done < <(find "$dir" -mindepth 1 -maxdepth 1 -print0 2>/dev/null) +# Empty dir $1 (/tmp), keeping the dir itself. +clean_tmp() { + find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null } -clean_tmp() { - local workdir="${RUNNER_WORKDIR:-/tmp/runner/work}" - local temp="${RUNNER_TEMP:-}" - workdir="${workdir%/}" - temp="${temp%/}" +cleanup() { + local before after + echo "[cleanup] Running post-job cleanup..." + before=$(df -h / | awk 'NR==2 {print $4}') - # Only a RUNNER_TEMP strictly under /tmp needs sparing. - case "$temp" in /tmp/*) ;; *) temp="" ;; esac + clean_tmp /tmp - clean_dir /tmp "$workdir" "$temp" -} + # Gradle daemon / journals / locks only. Keep modules-2, build-cache-*, + # transforms-*. $HOME covers both the root (default) and RUN_AS_ROOT=false + # (/home/runner) runner users. + rm -rf "$HOME"/.gradle/daemon/ "$HOME"/.gradle/caches/journal-* 2>/dev/null + # Shallow on purpose: Gradle's lock files sit at most three levels under + # caches/, and a find over the whole (multi-GB) tree is what made this slow. + find "$HOME"/.gradle/caches -maxdepth 3 -name '*.lock' -delete 2>/dev/null -clean_tmp + # Runner diagnostic logs + find /actions-runner/_diag -name '*.log' -delete 2>/dev/null -# Gradle daemon / journals / locks only. Keep modules-2, build-cache-*, -# transforms-*. $HOME covers both the root (default) and RUN_AS_ROOT=false -# (/home/runner) runner users. -rm -rf "$HOME"/.gradle/daemon/ 2>/dev/null || true -rm -rf "$HOME"/.gradle/caches/journal-* 2>/dev/null || true -# Shallow on purpose: Gradle's lock files sit at most three levels under -# caches/, and a find over the whole (multi-GB) tree is what made this slow. -find "$HOME"/.gradle/caches -maxdepth 3 -name '*.lock' -delete 2>/dev/null || true + # The job is over: the pruner may trim this slot again. + if [ -n "${RUNNER_SLOT_DIR:-}" ]; then rm -f "$RUNNER_SLOT_DIR/busy"; fi -# Runner diagnostic logs -find /actions-runner/_diag -name '*.log' -delete 2>/dev/null || true -find /runner-data/_diag -name '*.log' -delete 2>/dev/null || true + after=$(df -h / | awk 'NR==2 {print $4}') + echo "[cleanup] Post-job cleanup complete. Disk free: ${before} -> ${after}" +} -AFTER=$(df -h / | awk 'NR==2 {print $4}') -echo "[cleanup] Post-job cleanup complete. Disk free: ${BEFORE} -> ${AFTER}" +# Sourced (tests/): define the functions only. +[ "${BASH_SOURCE[0]}" = "$0" ] || return 0 +cleanup diff --git a/docker-compose.yml b/docker-compose.yml index 792cb11..12ad47d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,46 +1,63 @@ -# Three ephemeral runners, each with its own work and cache volumes, plus a -# `pruner` for maintenance. Never share a cache volume between runners: -# concurrent jobs would corrupt it. The action archive cache is the exception: -# read-only in every runner, filled by the pruner. `volumes:` lists replace on -# merge, so docker.sock is repeated per service. Small hosts can layer -# hosts/no-caches.compose.yml on top (see README). -# The environment is its own anchor so a service can change one key (LABELS) -# without restating the rest: a `<<:` merge replaces a whole mapping, so -# `environment:` in a service would otherwise drop every other variable. +# RUNNERS ephemeral runners, CONTROL_RUNNERS coordination runners and a +# `pruner` for maintenance. Size a host by setting RUNNERS in .env +# (hosts/sizing.sh suggests a value); there is nothing per runner to copy. +# +# State lives in one host directory, RUNNER_DATA_DIR, bind-mounted at the SAME +# path inside every container. On start each container claims a free slot, +# RUNNER_DATA_DIR/slots/, with a lock it holds for its whole life (see +# preflight-entrypoint.sh), and keeps its work dir and caches there: +# slots//work RUNNER_WORKDIR. The same path on the host, so the +# `docker run -v` / container actions a job hands +# to the host daemon see the checkout (issue #19). +# slots//cache/ bun, npm, pnpm, gradle, cargo, rustup, konan, +# mise, hk, pub and the hosted tool cache; the +# default home paths (/root/.bun, ...) link here. +# One slot per running container, so no two concurrent jobs share a cache. +# The action archive cache is the only shared state: read-only in the +# runners, filled by the pruner. x-runner-env: &runner-env - APP_ID: ${APP_ID} - APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} - APP_LOGIN: ${APP_LOGIN} - RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner} - RUNNER_WORKDIR: /tmp/runner/work - RUNNER_SCOPE: org - ORG_NAME: ${APP_LOGIN} - LABELS: ${LABELS:-docker} - RUNNER_GROUP: ${RUNNER_GROUP:-Default} - # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY - # non-empty value - including "false" - turns the flag on. - # - # EPHEMERAL: "true" is what these runners have always actually done: the old - # `EPHEMERAL: "false"` passed --ephemeral too. Each runner takes exactly one - # job, deregisters, exits, and `restart: unless-stopped` brings up a fresh - # container that registers again. That is intended: a clean container per - # job, while the named volumes below carry the caches across containers. - EPHEMERAL: "true" - # Empty (the default) leaves runner self-update on. Set it in .env to ANY - # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables - # them too. See .env.example. - DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} - # Ephemeral runners re-download every action archive for every job, and the - # few public IPs of these hosts hit codeload.github.com's 429 limit. The - # runner (>= 2.319) reads /_/.tar.gz before it - # downloads, and only reads: the `pruner` service fills the shared - # runner-action-cache volume (see pruner.sh), mounted read-only below. - ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE: /opt/actionarchivecache + APP_ID: ${APP_ID} + APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} + APP_LOGIN: ${APP_LOGIN} + RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner} + RUNNER_SCOPE: org + ORG_NAME: ${APP_LOGIN} + LABELS: ${LABELS:-docker} + RUNNER_GROUP: ${RUNNER_GROUP:-Default} + RUNNER_DATA_DIR: ${RUNNER_DATA_DIR:-/var/lib/github-runner} + RUNNER_CACHES: ${RUNNER_CACHES:-on} + # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY + # non-empty value - including "false" - turns the flag on. + # + # Each runner takes exactly one job, deregisters, exits, and + # `restart: unless-stopped` starts it again to register afresh. The slot's + # caches carry over; /tmp and the job's leftovers do not (cleanup.sh). + EPHEMERAL: "true" + # Empty (the default) leaves runner self-update on. Set it in .env to ANY + # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables + # them too. See .env.example. + DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} + # Ephemeral runners re-download every action archive for every job, and the + # few public IPs of these hosts hit codeload.github.com's 429 limit. The + # runner (>= 2.319) reads /_/.tar.gz before it + # downloads, and only reads: the `pruner` service fills the shared + # runner-action-cache volume (see pruner.sh), mounted read-only below. + ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE: /opt/actionarchivecache x-runner: &runner image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped + # .env is passed whole, so RUNNER__LABELS works for any slot id without + # listing each one here. Optional, so `docker compose config` runs without it. + env_file: + - path: .env + required: false environment: *runner-env + # Optional per-runner limits (hosts/sizing.sh suggests them; 0 = none). With + # a CPU limit the preflight sizes Gradle/cargo/make/cmake parallelism to it, + # so runners sharing a host don't each spawn a worker per host CPU. + cpus: ${RUNNER_CPUS:-0} + mem_limit: ${RUNNER_MEMORY:-0} # Runner.Listener is the long-lived process of a registered runner. Exec form # on purpose: under CMD-SHELL the wrapping `sh -c` carries the pattern in its # own command line and `pgrep -f` would match itself, healthy forever. The @@ -54,167 +71,58 @@ x-runner: &runner start_period: 120s volumes: - /var/run/docker.sock:/var/run/docker.sock + - ${RUNNER_DATA_DIR:-/var/lib/github-runner}:${RUNNER_DATA_DIR:-/var/lib/github-runner} + - runner-action-cache:/opt/actionarchivecache:ro security_opt: - label:disable -# Coordination runners carry only CONTROL_LABELS (default `control`), never a -# work label, so no build or test can land on them: they exist so the short -# jobs a merge waits on (a picker, a required summary) never queue behind long -# work. Those jobs read the API and check nothing out, so each service mounts -# no docker.sock and no tool cache, only its work directory and the read-only -# action cache. -x-control: &control - <<: *runner - profiles: [control] - environment: - <<: *runner-env - LABELS: ${CONTROL_LABELS:-control} - RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner}-control - services: - # Each runner registers with the host's LABELS, or with RUNNER__LABELS when - # that is set: one runner of a host can then join another pool (a `large` - # runner on a host whose others stay `light`) without a second host file. + # Slots 1..RUNNERS. Each registers with the host's LABELS, or with + # RUNNER__LABELS when that is set: one runner of a host can then join + # another pool (a `large` runner on a host whose others stay `light`). runner: <<: *runner - environment: - <<: *runner-env - LABELS: ${RUNNER_1_LABELS:-${LABELS:-docker}} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - runner-work:/tmp/runner/work - - runner-bun:/root/.bun - - runner-npm:/root/.npm - - runner-pnpm:/root/.cache/pnpm - - runner-gradle:/root/.gradle - - runner-cargo:/root/.cargo - - runner-rustup:/root/.rustup - - runner-konan:/root/.konan - - runner-mise-data:/root/.local/share/mise - - runner-mise-cache:/root/.cache/mise - - runner-hk:/root/.cache/hk - - runner-action-cache:/opt/actionarchivecache:ro - runner-2: - <<: *runner - environment: - <<: *runner-env - LABELS: ${RUNNER_2_LABELS:-${LABELS:-docker}} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-2:/tmp/runner/work - - runner-bun-2:/root/.bun - - runner-npm-2:/root/.npm - - runner-pnpm-2:/root/.cache/pnpm - - runner-gradle-2:/root/.gradle - - runner-cargo-2:/root/.cargo - - runner-rustup-2:/root/.rustup - - runner-konan-2:/root/.konan - - runner-mise-data-2:/root/.local/share/mise - - runner-mise-cache-2:/root/.cache/mise - - runner-hk-2:/root/.cache/hk - - runner-action-cache:/opt/actionarchivecache:ro - runner-3: + deploy: + replicas: ${RUNNERS:-3} + + # Coordination runners carry only CONTROL_LABELS (default `control`), never a + # work label, so no build or test can land on them: they exist so the short + # jobs a merge waits on (a picker, a required summary) never queue behind long + # work. Those jobs read the API and check nothing out, so they get no + # docker.sock and no tool caches (slots c1..cN, work dir only). + control: <<: *runner + deploy: + replicas: ${CONTROL_RUNNERS:-0} environment: <<: *runner-env - LABELS: ${RUNNER_3_LABELS:-${LABELS:-docker}} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-3:/tmp/runner/work - - runner-bun-3:/root/.bun - - runner-npm-3:/root/.npm - - runner-pnpm-3:/root/.cache/pnpm - - runner-gradle-3:/root/.gradle - - runner-cargo-3:/root/.cargo - - runner-rustup-3:/root/.rustup - - runner-konan-3:/root/.konan - - runner-mise-data-3:/root/.local/share/mise - - runner-mise-cache-3:/root/.cache/mise - - runner-hk-3:/root/.cache/hk - - runner-action-cache:/opt/actionarchivecache:ro - # Coordination runners (see x-control), off unless COMPOSE_PROFILES=control. - control: - <<: *control + LABELS: ${CONTROL_LABELS:-control} + RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner}-control + RUNNER_SLOT_PREFIX: c + RUNNER_CACHES: "off" volumes: - - runner-work-control:/tmp/runner/work + - ${RUNNER_DATA_DIR:-/var/lib/github-runner}:${RUNNER_DATA_DIR:-/var/lib/github-runner} - runner-action-cache:/opt/actionarchivecache:ro - control-2: - <<: *control - volumes: - - runner-work-control-2:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - # Maintenance off the job path (pruner.sh): nightly cache caps and docker - # prune, hourly action-cache fill. Mounts only what it caps. + + # Maintenance off the job path (pruner.sh): cache budgets per slot and docker + # prune, plus the hourly action-cache fill. pruner: image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped entrypoint: ["/usr/local/bin/pruner.sh"] + env_file: + - path: .env + required: false + # .env (env_file above) brings the App credentials it prefetches actions + # with and every pruner tunable. environment: - CACHES_ROOT: /caches - APP_ID: ${APP_ID} # App token for authenticated action downloads - APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} - APP_LOGIN: ${APP_LOGIN} - ACTION_CACHE_CAP_BYTES: ${ACTION_CACHE_CAP_BYTES:-2147483648} - PREFETCH_SOURCES: ${PREFETCH_SOURCES:-} - PREFETCH_EXTRA: ${PREFETCH_EXTRA:-} - PREFETCH_INTERVAL_SECONDS: ${PREFETCH_INTERVAL_SECONDS:-3600} - CACHE_CAP_BYTES: ${CACHE_CAP_BYTES:-21474836480} - PRUNE_INTERVAL_SECONDS: ${PRUNE_INTERVAL_SECONDS:-86400} + RUNNER_DATA_DIR: ${RUNNER_DATA_DIR:-/var/lib/github-runner} volumes: - /var/run/docker.sock:/var/run/docker.sock - - runner-gradle:/caches/1/gradle - - runner-bun:/caches/1/bun - - runner-mise-cache:/caches/1/mise-cache - - runner-hk:/caches/1/hk - - runner-mise-data:/caches/1/mise-data - - runner-gradle-2:/caches/2/gradle - - runner-bun-2:/caches/2/bun - - runner-mise-cache-2:/caches/2/mise-cache - - runner-hk-2:/caches/2/hk - - runner-mise-data-2:/caches/2/mise-data - - runner-gradle-3:/caches/3/gradle - - runner-bun-3:/caches/3/bun - - runner-mise-cache-3:/caches/3/mise-cache - - runner-hk-3:/caches/3/hk - - runner-mise-data-3:/caches/3/mise-data + - ${RUNNER_DATA_DIR:-/var/lib/github-runner}:${RUNNER_DATA_DIR:-/var/lib/github-runner} - runner-action-cache:/opt/actionarchivecache security_opt: - label:disable volumes: runner-action-cache: {} # read-only in runners, filled by the pruner - runner-work: {} - runner-work-2: {} - runner-work-3: {} - runner-work-control: {} - runner-work-control-2: {} - runner-bun: {} - runner-bun-2: {} - runner-bun-3: {} - runner-npm: {} - runner-npm-2: {} - runner-npm-3: {} - runner-pnpm: {} - runner-pnpm-2: {} - runner-pnpm-3: {} - runner-gradle: {} - runner-gradle-2: {} - runner-gradle-3: {} - runner-cargo: {} - runner-cargo-2: {} - runner-cargo-3: {} - runner-rustup: {} - runner-rustup-2: {} - runner-rustup-3: {} - runner-konan: {} - runner-konan-2: {} - runner-konan-3: {} - runner-mise-data: {} - runner-mise-data-2: {} - runner-mise-data-3: {} - runner-mise-cache: {} - runner-mise-cache-2: {} - runner-mise-cache-3: {} - runner-hk: {} - runner-hk-2: {} - runner-hk-3: {} diff --git a/hosts/no-caches.compose.yml b/hosts/no-caches.compose.yml deleted file mode 100644 index 4f7cc25..0000000 --- a/hosts/no-caches.compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -# For small hosts (e.g. ARM64 boards with little disk): no per-runner cache -# volumes, only docker.sock, the work volume and the action archive cache. -# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml (in .env) -# `!override` replaces the base volume lists instead of merging into them. -services: - runner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - runner-2: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-2:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - runner-3: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-3:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - pruner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-action-cache:/opt/actionarchivecache diff --git a/hosts/sizing.sh b/hosts/sizing.sh new file mode 100755 index 0000000..339256c --- /dev/null +++ b/hosts/sizing.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Suggest .env settings for this host. Read-only: it inspects the machine and +# prints a snippet to paste into .env next to docker-compose.yml. +# +# hosts/sizing.sh [light|large] +# +# Profile = what one runner should get: +# light 1 CPU, 3 GiB checks, lint, Next.js/bun builds (default) +# large 2 CPU, 6 GiB Gradle/Android and other heavy builds +# Override with RUNNER_CPUS / RUNNER_MEM_GIB. The host keeps HOST_RESERVE_GIB +# (default 2) of memory for itself and the docker daemon. +# +# Linux only (reads /proc and uses GNU df). Run it on the host, not in a +# runner container. +set -euo pipefail + +profile="${1:-light}" +case "$profile" in + light) cpus="${RUNNER_CPUS:-1}" mem="${RUNNER_MEM_GIB:-3}" ;; + large) cpus="${RUNNER_CPUS:-2}" mem="${RUNNER_MEM_GIB:-6}" ;; + *) echo "usage: $0 [light|large]" >&2; exit 2 ;; +esac +reserve="${HOST_RESERVE_GIB:-2}" +data_dir="${RUNNER_DATA_DIR:-/var/lib/github-runner}" + +nproc=$(nproc) +mem_gib=$(awk '/^MemTotal:/ {printf "%d", $2 / 1048576}' /proc/meminfo) + +# Size the disk that will hold the data dir (its nearest existing parent). +probe="$data_dir" +while [ ! -e "$probe" ]; do probe=$(dirname "$probe"); done +read -r disk_bytes avail_bytes < <(df -B1 --output=size,avail "$probe" | awk 'NR==2 {print $1, $2}') + +by_cpu=$((nproc / cpus)) +by_mem=$(((mem_gib - reserve) / mem)) +runners=$((by_cpu < by_mem ? by_cpu : by_mem)) +[ "$runners" -ge 1 ] || runners=1 + +# The pruner's own budget (CACHE_BUDGET_BYTES / CACHE_BUDGET_PCT, honoring +# them when set), sized against this disk. Printed so the per-runner share is +# visible. pruner.sh only defines functions when sourced. +# shellcheck source=SCRIPTDIR/../pruner.sh +source "$(dirname "$0")/../pruner.sh" +budget=$(RUNNER_DATA_DIR="$probe" slot_budget 1) +gib() { awk -v b="$1" 'BEGIN {printf "%.1f", b / 1073741824}'; } + +arch=$(uname -m) +kvm=no +if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then kvm=yes; fi +docker_version=$(docker version --format '{{.Server.Version}}' 2>/dev/null || echo unknown) + +cat < by cpu $by_cpu, by memory $by_mem +RUNNERS=$runners +RUNNER_DATA_DIR=$data_dir +# Per-runner limits keep one job from starving the others; the preflight then +# caps Gradle/cargo/make/cmake workers to RUNNER_CPUS. +RUNNER_CPUS=$cpus +RUNNER_MEMORY=${mem}g +# cache budget: $(gib "$budget") GiB for the host, ~$(gib $((budget / runners))) GiB per slot +EOF +case "$arch" in + aarch64|arm64) + echo "# arm64: the Android SDK's build tools are x86_64-only, so use the slim image" + echo "RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner:slim" ;; +esac +if [ "$runners" -eq 1 ] && [ "$budget" -lt $((10 * 1024 * 1024 * 1024)) ]; then + echo "# little disk: consider RUNNER_CACHES=off" +fi diff --git a/pre-job.sh b/pre-job.sh index 3149263..a94b941 100755 --- a/pre-job.sh +++ b/pre-job.sh @@ -2,9 +2,14 @@ # Pre-job hook for GitHub Actions runner. # Invoked via ACTIONS_RUNNER_HOOK_JOB_STARTED before each job. # -# JOB_COMPLETED does not run when the runner process dies (OOM). This sweep -# covers that case: delete incomplete/poison entries only. Do not rm -rf -# cache trees — named volumes are the warm cache. +# 1. Mark the slot busy so pruner.sh leaves its caches alone while the job +# runs. If the pruner is mid-pass on this slot, wait for it (bounded) so a +# job never starts on a half-evicted cache. +# 2. Crash sweep. JOB_COMPLETED (cleanup.sh) does not run when the runner +# process dies (OOM), and a `busy` marker it never removed says so. Then, +# and only then, delete what a killed job leaves half-written. Do not rm -rf +# cache trees: the slot caches are the warm cache. Without a slot (the image +# run outside compose) there is no marker, so it always sweeps. # # Intentionally not using set -e: individual prune failures should not # prevent the job from starting. @@ -15,17 +20,35 @@ # exact cache key (no restore-keys) so a corrupt entry cannot be carried # forward across lockfile changes. -echo "[pre-job] Surgical prune of incomplete cache entries..." +PRUNE_WAIT_SECONDS="${PRUNE_WAIT_SECONDS:-600}" -# Bun: a killed job leaves partial tarballs (*.tmp / *.part). The next job -# then fails with "Fail extracting tarball for ..." and it looks like a -# dependency problem rather than a runner problem. -find "$HOME"/.bun/install/cache \( -name '*.tmp' -o -name '*.part' \) -delete 2>/dev/null || true +crashed=true +if [ -n "${RUNNER_SLOT_DIR:-}" ] && [ -d "$RUNNER_SLOT_DIR" ]; then + [ -e "$RUNNER_SLOT_DIR/busy" ] || crashed=false + if exec 8>"$RUNNER_SLOT_DIR/.prune.lock"; then + flock -w "$PRUNE_WAIT_SECONDS" 8 || + echo "[pre-job] pruner still busy on $RUNNER_SLOT_DIR after ${PRUNE_WAIT_SECONDS}s; starting anyway" + touch "$RUNNER_SLOT_DIR/busy" + exec 8>&- + fi +fi + +[ "$crashed" = true ] || exit 0 +echo "[pre-job] Previous job did not finish; sweeping incomplete cache entries..." + +# Bun: a killed job leaves partial extractions (*.tmp / *.part, and the .tmp +# scratch dir) next to the package folders: at the top of the cache, or one +# level down for @scope. The next job then fails with "Fail extracting tarball +# for ..." and it looks like a dependency problem rather than a runner +# problem. Only those two levels: the cache holds hundreds of thousands of +# files, and a file inside a package that happens to end in .tmp is content. +bun_cache="$HOME/.bun/install/cache" +find "$bun_cache" "$bun_cache"/@*/ -mindepth 1 -maxdepth 1 \( -name '*.tmp' -o -name '*.part' \) \ + -exec rm -rf {} + 2>/dev/null # Gradle: leftover lock / journal / daemon state from a killed compile. # Keep modules-2, build-cache-*, and transforms-*. -rm -f "$HOME"/.gradle/caches/modules-2/modules-2.lock 2>/dev/null || true -rm -rf "$HOME"/.gradle/caches/journal-* 2>/dev/null || true -rm -rf "$HOME"/.gradle/daemon/ 2>/dev/null || true +rm -f "$HOME"/.gradle/caches/modules-2/modules-2.lock 2>/dev/null +rm -rf "$HOME"/.gradle/caches/journal-* "$HOME"/.gradle/daemon/ 2>/dev/null -echo "[pre-job] Prune complete." +echo "[pre-job] Sweep complete." diff --git a/preflight-entrypoint.sh b/preflight-entrypoint.sh index 5f5b013..9fea950 100644 --- a/preflight-entrypoint.sh +++ b/preflight-entrypoint.sh @@ -1,7 +1,21 @@ #!/bin/bash -# Preflight wrapper around the upstream myoung34 entrypoint. +# Preflight wrapper around the upstream myoung34 entrypoint. Two jobs: # -# When CONFIGURED_ACTIONS_RUNNER_FILES_DIR is set, the upstream entrypoint +# 1. Slot claim (when RUNNER_DATA_DIR is set, as docker-compose.yml does). +# Every runner container of a host shares RUNNER_DATA_DIR, bind-mounted at +# the same path on host and container. This claims the lowest free +# slots/ with flock on its .lock and keeps the lock +# for the container's whole life (fd 9 survives `exec`), so a slot's work +# dir and caches are only ever used by one container at a time. Then: +# - RUNNER_WORKDIR = slots//work. It is the same path on the host, so +# `docker run -v "$PWD:..."` and container actions mount the checkout. +# - LABELS = RUNNER__LABELS when that is set (numeric slots only). +# - The tools' default cache paths link into slots//cache/, +# unless RUNNER_CACHES=off (small hosts, control runners). +# pre-job.sh / cleanup.sh mark the slot busy for pruner.sh through +# RUNNER_SLOT_DIR. +# +# 2. Registration check. When CONFIGURED_ACTIONS_RUNNER_FILES_DIR is set, the upstream entrypoint # copies the persisted config into /actions-runner and skips registration # whenever a .runner file exists — with no validation of the stored # credentials and no fallback to ACCESS_TOKEN. If the runner was meanwhile @@ -21,6 +35,125 @@ log() { echo "[preflight] $*"; } +# store:path pairs. The path is the tool's default location for root (jobs run +# as root, HOME=/root); pruner.sh knows how to evict each store safely. +CACHE_LINKS=( + bun:/root/.bun + npm:/root/.npm + pnpm/store:/root/.local/share/pnpm # store-dir default: one store, one unit + pnpm/cache:/root/.cache/pnpm # metadata cache + gradle:/root/.gradle + cargo:/root/.cargo + rustup:/root/.rustup + konan:/root/.konan + mise-data:/root/.local/share/mise + mise-cache:/root/.cache/mise + hk:/root/.cache/hk + pub:/root/.pub-cache + toolcache:/opt/hostedtoolcache # AGENT_TOOLSDIRECTORY: setup-* actions, Flutter +) + +# Claim the lowest free slot under $1 into SLOT_ID. fd 9 stays open on the +# claimed .lock in this shell, which then execs the runner with it. +claim_slot() { + local root="$1" prefix="${RUNNER_SLOT_PREFIX:-}" n=1 id + while :; do + id="${prefix}${n}" + mkdir -p "$root/slots/$id" || return 1 + exec 9>"$root/slots/$id/.lock" || return 1 + if flock -n 9; then + SLOT_ID="$id" + return 0 + fi + exec 9>&- + n=$((n + 1)) + done +} + +# Point each default cache path at the slot. Anything already at the path +# (the container's writable layer from an earlier run, or an empty image dir) +# is replaced: the slot is the cache. +# Restarts (every job) find the links in place and fork nothing. +link_caches() { + local cache="$1" pair store path + for pair in "${CACHE_LINKS[@]}"; do + store="${pair%%:*}" + path="${pair#*:}" + [ -L "$path" ] && [ "$path" -ef "$cache/$store" ] && continue + mkdir -p "$cache/$store" "${path%/*}" + rm -rf "$path" + ln -s "$cache/$store" "$path" + done +} + +# CPUs this container may use: the cgroup v2 quota (cpu.max, from compose +# `cpus:`) rounded up, or nothing when unlimited. `nproc` inside a container +# reports the host's CPUs whatever the quota, and the tools size their worker +# pools from it. +cpu_quota() { + local quota period + read -r quota period < /sys/fs/cgroup/cpu.max 2>/dev/null || return 1 + if [ "$quota" = max ] || [ "${period:-0}" -le 0 ]; then return 1; fi + echo $(((quota + period - 1) / period)) +} + +# Under a CPU limit, cap the build tools' parallelism to it, unless the job +# environment already chose. The JVM and Node read the cgroup themselves. +tune_parallelism() { + local n + n=$(cpu_quota) || return 0 + export RUNNER_CPU_LIMIT="$n" + export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$n}" + export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-$n}" + export MAKEFLAGS="${MAKEFLAGS:--j$n}" + case "${GRADLE_OPTS:-}" in + *org.gradle.workers.max*) ;; + *) export GRADLE_OPTS="${GRADLE_OPTS:+$GRADLE_OPTS }-Dorg.gradle.workers.max=$n" ;; + esac + log "cpu limit $n: build parallelism capped to $n" +} + +# Install rustup (no toolchain) into ~/.cargo, which is the slot's cargo cache +# when caches are on, from the image's rustup-init. Offline and done once per +# slot: later starts find it. Toolchains then persist in ~/.rustup. +ensure_rustup() { + [ -x /usr/local/bin/rustup-init ] || return 0 + [ -x "$HOME/.cargo/bin/rustup" ] && return 0 + # Self-update off: the image pins rustup, and the check is a network call on + # every toolchain install. + if rustup-init -y -q --no-modify-path --default-toolchain none >/dev/null 2>&1 && + "$HOME/.cargo/bin/rustup" set auto-self-update disable >/dev/null 2>&1; then + log "rustup installed into $HOME/.cargo" + else + log "rustup-init failed; jobs install rustup themselves" + fi +} + +setup_slot() { + local root="${RUNNER_DATA_DIR:-}" id dir labels_var + [ -n "$root" ] || return 0 + # Not in $(...): a subshell would close fd 9, and with it the claim. + if ! claim_slot "$root"; then + log "cannot claim a slot under $root/slots; running without one" + return 0 + fi + id="$SLOT_ID" + dir="$root/slots/$id" + mkdir -p "$dir/work" + export RUNNER_SLOT="$id" RUNNER_SLOT_DIR="$dir" RUNNER_WORKDIR="$dir/work" + export RUNNER_NAME_PREFIX="${RUNNER_NAME_PREFIX:-runner}-${id}" + labels_var="RUNNER_${id}_LABELS" + if [[ "$id" =~ ^[0-9]+$ ]] && [ -n "${!labels_var:-}" ]; then + export LABELS="${!labels_var}" + fi + if [ "${RUNNER_CACHES:-on}" = off ]; then + log "slot $id ($dir), caches off" + else + link_caches "$dir/cache" + log "slot $id ($dir), caches under $dir/cache" + fi +} + # The runner's IsConfigured()/HasCredentials() checks treat .runner_migrated / # .credentials_migrated as equivalent to the primary files, so a wipe must # remove all five or config.sh refuses to re-register while run.sh can't load @@ -118,5 +251,11 @@ preflight() { esac } +# Sourced (tests/): define the functions only. +[ "${BASH_SOURCE[0]}" = "$0" ] || return 0 + +setup_slot +ensure_rustup +tune_parallelism preflight exec /entrypoint.sh "$@" diff --git a/tests/slots.sh b/tests/slots.sh new file mode 100644 index 0000000..09ae334 --- /dev/null +++ b/tests/slots.sh @@ -0,0 +1,210 @@ +#!/bin/bash +# Slot claim, cache links and the job hooks (preflight-entrypoint.sh, +# pre-job.sh, cleanup.sh) against a throwaway RUNNER_DATA_DIR: +# - concurrent claims get distinct slots; a released slot is reused; +# - RUNNER__LABELS applies to its slot only; control slots are c; +# - cache paths link into the slot (replacing what was there), not with +# RUNNER_CACHES=off; +# - pre-job marks the slot busy, waits for a pruner holding .prune.lock, +# and sweeps top-level bun leftovers only after a crashed job; cleanup +# clears busy and /tmp but never the work dir; +# - every linked cache store is one the pruner bounds. +# Needs flock and GNU find (Linux, or `docker run ubuntu`): +# bash tests/slots.sh +# Each ( ... ) group stands for one runner container, so the runner env it +# sets is meant to stay inside it: +# shellcheck disable=SC2030,SC2031,SC2034 +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } + +data="$work/data" +home="$work/home" +mkdir -p "$data" "$home" + +# Run setup_slot in a child that holds its slot for $2 seconds and reports +# " " to file $1. +claim() { + local out="$1" hold="$2" + ( + # shellcheck source=SCRIPTDIR/../preflight-entrypoint.sh + source "$here/preflight-entrypoint.sh" + CACHE_LINKS=("bun:$home/.bun" "pnpm/store:$home/.local/share/pnpm" "toolcache:$home/toolcache") + RUNNER_DATA_DIR="$data" setup_slot >/dev/null + echo "$RUNNER_SLOT $LABELS $RUNNER_WORKDIR" >"$out" + sleep "$hold" + ) +} + +# --- concurrent claims ------------------------------------------------------- +export LABELS=docker,light RUNNER_2_LABELS=docker,light,large +for i in 1 2 3 4; do claim "$work/claim.$i" 3 & done +sleep 1 +ids=$(cut -d' ' -f1 "$work"/claim.* | sort -n | tr '\n' ' ') +[ "$ids" = "1 2 3 4 " ] || fail "concurrent claims got '$ids', want '1 2 3 4 '" +for f in "$work"/claim.*; do + read -r id labels wd <"$f" + [ "$wd" = "$data/slots/$id/work" ] || fail "slot $id workdir $wd" + [ -d "$wd" ] || fail "slot $id workdir missing" + if [ "$id" = 2 ]; then + [ "$labels" = docker,light,large ] || fail "slot 2 labels $labels" + else + [ "$labels" = docker,light ] || fail "slot $id labels $labels" + fi +done +wait +unset RUNNER_2_LABELS + +# Released slots are claimed again from the lowest id. +claim "$work/again" 0 +[ "$(cut -d' ' -f1 "$work/again")" = 1 ] || fail "after release, claim got $(cat "$work/again")" + +# --- cache links --------------------------------------------------------------- +[ "$(readlink "$home/.bun")" = "$data/slots/1/cache/bun" ] || fail "bun link: $(readlink "$home/.bun")" +[ "$(readlink "$home/.local/share/pnpm")" = "$data/slots/1/cache/pnpm/store" ] || fail "pnpm store link" +[ -d "$data/slots/1/cache/toolcache" ] || fail "toolcache store not created" + +# An existing directory at a cache path (the writable layer) is replaced. +rm "$home/toolcache" && mkdir -p "$home/toolcache/stale" +claim "$work/replace" 0 +[ -L "$home/toolcache" ] || fail "existing dir not replaced by a link" +[ -e "$data/slots/1/cache/toolcache/stale" ] && fail "stale layer content leaked into the slot" + +# RUNNER_CACHES=off leaves the paths alone. +rm "$home/.bun" +( + # shellcheck source=SCRIPTDIR/../preflight-entrypoint.sh + source "$here/preflight-entrypoint.sh" + CACHE_LINKS=("bun:$home/.bun") + RUNNER_CACHES=off + RUNNER_DATA_DIR="$data" setup_slot >/dev/null +) +[ -e "$home/.bun" ] && fail "RUNNER_CACHES=off still linked caches" + +# Control runners claim c slots and keep their LABELS. +( + # shellcheck source=SCRIPTDIR/../preflight-entrypoint.sh + source "$here/preflight-entrypoint.sh" + CACHE_LINKS=() + RUNNER_SLOT_PREFIX=c LABELS=control RUNNER_1_LABELS=nope + RUNNER_DATA_DIR="$data" setup_slot >/dev/null + echo "$RUNNER_SLOT $LABELS" >"$work/control" +) +[ "$(cat "$work/control")" = "c1 control" ] || fail "control claim: $(cat "$work/control")" + +# No RUNNER_DATA_DIR: no slot, nothing exported (the image outside compose). +( + # shellcheck source=SCRIPTDIR/../preflight-entrypoint.sh + source "$here/preflight-entrypoint.sh" + unset RUNNER_DATA_DIR + setup_slot + [ -z "${RUNNER_SLOT:-}" ] || echo "slot without data dir" >"$work/noslot" +) +[ -e "$work/noslot" ] && fail "claimed a slot without RUNNER_DATA_DIR" + +# --- parallelism under a CPU limit ---------------------------------------------- +# cpu_quota reads the cgroup; stub it with what a 1.5-CPU limit gives. +( + # shellcheck source=SCRIPTDIR/../preflight-entrypoint.sh + source "$here/preflight-entrypoint.sh" + cpu_quota() { echo 2; } + unset CARGO_BUILD_JOBS CMAKE_BUILD_PARALLEL_LEVEL MAKEFLAGS + GRADLE_OPTS="-Xmx1g" MAKEFLAGS="-j9" + tune_parallelism >/dev/null + echo "$RUNNER_CPU_LIMIT|$CARGO_BUILD_JOBS|$CMAKE_BUILD_PARALLEL_LEVEL|$MAKEFLAGS|$GRADLE_OPTS" >"$work/tuned" +) +[ "$(cat "$work/tuned")" = "2|2|2|-j9|-Xmx1g -Dorg.gradle.workers.max=2" ] || + fail "cpu limit tuned to: $(cat "$work/tuned")" +( + source "$here/preflight-entrypoint.sh" + cpu_quota() { return 1; } # unlimited + unset CARGO_BUILD_JOBS + tune_parallelism >/dev/null + echo "${CARGO_BUILD_JOBS:-unset}" >"$work/tuned" +) +[ "$(cat "$work/tuned")" = unset ] || fail "no cpu limit still tuned: $(cat "$work/tuned")" + +# --- pre-job --------------------------------------------------------------- +slot="$data/slots/1" +cache="$home/.bun/install/cache" +bun_leftovers() { + mkdir -p "$cache/.tmp/x" "$cache/next@16.3.8@@@1" "$cache/@scope/pkg@1@@@1" + : >"$cache/next@16.3.8@@@1/chunk.tmp" # package content that ends in .tmp + : >"$cache/react@19.part" + : >"$cache/@scope/pkg@2.tmp" +} + +# A clean previous job (no busy marker): mark busy, sweep nothing. +bun_leftovers +rm -f "$slot/busy" +HOME="$home" RUNNER_SLOT_DIR="$slot" bash "$here/pre-job.sh" >/dev/null +[ -e "$slot/busy" ] || fail "pre-job did not mark the slot busy" +[ -e "$cache/react@19.part" ] || fail "pre-job swept after a job that finished" + +# A crashed previous job left busy behind: sweep the leftovers. A pruner +# holding .prune.lock delays the job, up to PRUNE_WAIT_SECONDS. +(exec 8>"$slot/.prune.lock"; flock 8; sleep 2) & +sleep 0.3 +start=$(date +%s) +HOME="$home" RUNNER_SLOT_DIR="$slot" PRUNE_WAIT_SECONDS=10 bash "$here/pre-job.sh" >/dev/null +waited=$(($(date +%s) - start)) +wait +[ "$waited" -ge 1 ] || fail "pre-job did not wait for the pruner (${waited}s)" +[ -e "$cache/.tmp" ] && fail "bun .tmp scratch dir survived" +[ -e "$cache/react@19.part" ] && fail "top-level .part survived" +[ -e "$cache/@scope/pkg@2.tmp" ] && fail "scoped .tmp survived" +[ -e "$cache/next@16.3.8@@@1/chunk.tmp" ] || fail "pre-job deleted package content" + +# Bounded: a pruner that never lets go does not block the job. +(exec 8>"$slot/.prune.lock"; flock 8; sleep 4) & +sleep 0.3 +start=$(date +%s) +HOME="$home" RUNNER_SLOT_DIR="$slot" PRUNE_WAIT_SECONDS=1 bash "$here/pre-job.sh" >/dev/null +waited=$(($(date +%s) - start)) +[ "$waited" -le 3 ] || fail "pre-job waited ${waited}s past PRUNE_WAIT_SECONDS=1" +wait + +# --- cleanup --------------------------------------------------------------- +tmp="$work/tmp" +mkdir -p "$tmp/junk/deep" "$slot/work/repo/repo" +: >"$tmp/file" +: >"$slot/work/repo/repo/keep" +( + # shellcheck source=SCRIPTDIR/../cleanup.sh + source "$here/cleanup.sh" + set +eu # cleanup.sh runs without -e on purpose: one failed step must not stop the rest + clean_tmp "$tmp" + clean_tmp() { :; } # the real /tmp is not ours to wipe + HOME="$home" RUNNER_SLOT_DIR="$slot" cleanup >/dev/null +) +left=$(find "$tmp" -mindepth 1 -maxdepth 1 -printf '%f ') +[ -z "$left" ] || fail "cleanup left ${left}in tmp" +[ -d "$tmp" ] || fail "cleanup removed tmp itself" +[ -e "$slot/work/repo/repo/keep" ] || fail "cleanup touched the work dir" +[ -e "$slot/busy" ] && fail "cleanup did not clear busy" + +# --- store lists ------------------------------------------------------------ +# Every store the preflight links must be one the pruner bounds. +( + # shellcheck source=SCRIPTDIR/../pruner.sh + source "$here/pruner.sh" + pruned=" $(slot_stores | tr '\n' ' ')" + source "$here/preflight-entrypoint.sh" + for pair in "${CACHE_LINKS[@]}"; do + store="${pair%%:*}" + store="${store%%/*}" + case "$pruned" in *" $store "*) ;; *) echo "$store" ;; esac + done +) >"$work/unpruned" +[ -s "$work/unpruned" ] && fail "linked but never pruned: $(tr '\n' ' ' <"$work/unpruned")" + +if [ "$fails" -gt 0 ]; then + echo "$fails check(s) failed" + exit 1 +fi +echo "slots: all checks passed" From 99b99c61aa6c2f118b41380f80cb6e48c36ba3d2 Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:04:17 +0300 Subject: [PATCH 09/11] feat(image): slim/default/ndk targets, shared SDK layer, opt-in NDK, xxd and rustup - sdk stage runs sdkmanager on a fixed JDK, so the SDK layer is identical for every JDK/Node tag - core: base + one apt layer (xxd, pigz) + rustup-init + JDK + Node; runner-slim = core without the Android SDK - android = core + SDK on top: the weekly SDK bump replaces only that layer; runner (default, last) and runner-ndk build on it - NDK and cmake move to the opt-in runner-ndk target: default image 5.65 -> 3.18 GiB, slim 2.58 GiB on arm64 (#17) - runtime scripts come from a scratch stage copied last, so script edits never invalidate the SDK/NDK layers - rustup without a toolchain; the preflight sets it up per slot so toolchains persist in the slot cache Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 294 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 201 insertions(+), 93 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a493e0..aac655a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,88 +6,146 @@ ARG BUILD_TOOLS=37.0.0 ARG NDK_VERSION=30.0.16248370 ARG SDK_TOOLS=8512546_latest ARG ANDROID_ROOT=/usr/local/lib/android +# JDK that sdkmanager runs on in the sdk stages. Fixed on purpose and independent +# of JAVA_VERSION: the SDK layer must come out identical for every JDK/Node tag, +# so it is shared in the registry and on hosts instead of stored once per tag. +ARG SDK_JAVA_VERSION=21 ################################################################################ -# base image from https://github.com/myoung34/docker-github-actions-runner -# - install temurin java into it -################################################################################ -FROM myoung34/github-runner:$VERSION AS java -ARG JAVA_VERSION -ARG NODE_VERSION -# TARGETARCH is set automatically by Buildx (e.g. amd64, arm64) and matches the -# arch suffix Temurin uses on its apt-installed JDK directory. -ARG TARGETARCH - -# https://adoptium.net/installation/linux -# add the gpp key and apt repo -RUN mkdir -p /etc/apt/keyrings && \ - wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc && \ - echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list - -# install the temurin jdk -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends temurin-$JAVA_VERSION-jdk -ENV JAVA_HOME=/usr/lib/jvm/temurin-$JAVA_VERSION-jdk-$TARGETARCH - -# install ip tools -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iproute2 - -# install cmake build-essential -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cmake build-essential swig - -# install nodejs from NodeSource. Replaces the base image's Ubuntu-default node -# (currently node 18 on noble) with a current LTS that user workflows can rely -# on without an extra setup-node step. -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends nodejs \ - && rm -rf /var/lib/apt/lists/* - -################################################################################ -# - install android sdk +# sdk: the Android SDK, built once for every tag +# - runs on a fixed JDK (SDK_JAVA_VERSION), never on JAVA_VERSION/NODE_VERSION +# - only the build args below reach this stage, so a JDK or Node bump does not +# invalidate it ################################################################################ -FROM java AS android +FROM eclipse-temurin:${SDK_JAVA_VERSION}-jdk-noble AS sdk ARG COMPILE_SDK ARG BUILD_TOOLS -ARG NDK_VERSION ARG SDK_TOOLS ARG ANDROID_ROOT -WORKDIR /tmp - -# download android tools and use it to install the SDK -RUN mkdir -p ${ANDROID_ROOT}/sdk/cmdline-tools/latest -RUN wget -O android-sdk.zip https://dl.google.com/android/repository/commandlinetools-linux-$SDK_TOOLS.zip && unzip android-sdk.zip && cp -r ./cmdline-tools/* ${ANDROID_ROOT}/sdk/cmdline-tools/latest -RUN ${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager --licenses >/dev/null +RUN apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends unzip wget \ + && rm -rf /var/lib/apt/lists/* -# Install latest Android SDK components only. Older versions are not needed since -# build-tools and NDK are backward compatible, and you only need the platform -# matching your project's compileSdk. Projects needing older platforms can install -# them at build time via sdkmanager. -# # sdkmanager has no download retry: a truncated download from dl.google.com # surfaces as "Archive is not a ZIP archive" and fails the whole build. It is # idempotent (already-installed packages are skipped), so retry the whole # command a few times, clearing its download scratch dir between attempts. -RUN attempt=1; \ - until echo "y" | ${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager --sdk_root=$ANDROID_ROOT/sdk/ \ +# Usage: sdk-install ... +RUN cat > /usr/local/bin/sdk-install <<'EOF' && chmod +x /usr/local/bin/sdk-install +#!/bin/sh +set -u +sdk_root=$1 +shift +attempt=1 +until echo "y" | "${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager" --sdk_root="$sdk_root" "$@"; do + if [ "$attempt" -ge 3 ]; then + echo "sdkmanager failed after $attempt attempts" >&2 + exit 1 + fi + echo "sdkmanager attempt $attempt failed, retrying in 15s..." >&2 + rm -rf "$sdk_root/.temp" + attempt=$((attempt + 1)) + sleep 15 +done +EOF + +# download android tools and use it to install the SDK +WORKDIR /tmp +RUN mkdir -p ${ANDROID_ROOT}/sdk/cmdline-tools/latest \ + && wget -q -O android-sdk.zip https://dl.google.com/android/repository/commandlinetools-linux-$SDK_TOOLS.zip \ + && unzip -q android-sdk.zip \ + && cp -r ./cmdline-tools/* ${ANDROID_ROOT}/sdk/cmdline-tools/latest \ + && rm -rf android-sdk.zip cmdline-tools +RUN ${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager --licenses >/dev/null + +# Install latest Android SDK components only. Older versions are not needed since +# build-tools are backward compatible, and you only need the platform matching +# your project's compileSdk. Projects needing older platforms can install them at +# build time via sdkmanager. The NDK and cmake live in the opt-in sdk-ndk stage. +RUN sdk-install ${ANDROID_ROOT}/sdk \ "platform-tools" \ "platforms;android-${COMPILE_SDK}" \ "build-tools;${BUILD_TOOLS}" \ + && rm -rf ${ANDROID_ROOT}/sdk/.temp + +################################################################################ +# sdk-ndk: NDK and the SDK's own cmake, for the opt-in runner-ndk target +# - installed into a separate sdk root, so the final image gets only the delta +# (ndk/ and cmake/) instead of a second full copy of the SDK +################################################################################ +FROM sdk AS sdk-ndk +ARG NDK_VERSION +ARG ANDROID_ROOT + +RUN mkdir -p /ndk-sdk && cp -r ${ANDROID_ROOT}/sdk/licenses /ndk-sdk/licenses +RUN sdk-install /ndk-sdk \ "ndk;${NDK_VERSION}" \ - "cmake;3.22.1"; do \ - if [ "$attempt" -ge 3 ]; then echo "sdkmanager failed after $attempt attempts" >&2; exit 1; fi; \ - echo "sdkmanager attempt $attempt failed, retrying in 15s..." >&2; \ - rm -rf ${ANDROID_ROOT}/sdk/.temp; \ - attempt=$((attempt + 1)); \ - sleep 15; \ - done + "cmake;3.22.1" \ + && rm -rf /ndk-sdk/.temp -WORKDIR /actions-runner -ENV PATH="${PATH}:/usr/local/lib/android/sdk/platform-tools/" +################################################################################ +# core: everything every runner target shares (and, as is, runner-slim) +# base image from https://github.com/myoung34/docker-github-actions-runner +# - the tag-independent apt packages first, then the JDK, then Node +################################################################################ +FROM myoung34/github-runner:$VERSION AS core +# Packages that do not depend on JAVA_VERSION/NODE_VERSION, so this layer is +# shared by every tag too. +# - iproute2: ip tools +# - cmake, build-essential, swig: native builds +# - xxd: not in the base image +# - pigz: parallel gzip, used by tar/docker when available +RUN apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + iproute2 cmake build-essential swig xxd pigz \ + && rm -rf /var/lib/apt/lists/* -# Set env variable for SDK Root (https://developer.android.com/studio/command-line/variables) -# ANDROID_HOME is deprecated, but older versions of Gradle rely on it -ENV ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk -ENV ANDROID_HOME=$ANDROID_ROOT/sdk +# rustup-init only, no toolchain: ~/.cargo and ~/.rustup are links into the +# runner's slot, so anything installed there at build time would be hidden. +# The preflight runs it once per slot (`--default-toolchain none`, offline), so +# `rustup` is on PATH and the toolchains a job installs (dtolnay/rust-toolchain, +# rust-toolchain.toml) land in the warm slot cache. The checksum comes from the +# same versioned archive. +ARG RUSTUP_VERSION=1.29.1 +# TARGETARCH is set automatically by Buildx (e.g. amd64, arm64). It also matches +# the arch suffix Temurin uses on its apt-installed JDK directory (JAVA_HOME). +ARG TARGETARCH +RUN case "$TARGETARCH" in \ + amd64) triple=x86_64-unknown-linux-gnu ;; \ + arm64) triple=aarch64-unknown-linux-gnu ;; \ + *) echo "unsupported arch $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${triple}/rustup-init" \ + && curl -fsSL --retry 3 -o /usr/local/bin/rustup-init "$url" \ + && echo "$(curl -fsSL --retry 3 "$url.sha256" | cut -d' ' -f1) /usr/local/bin/rustup-init" | sha256sum -c - \ + && chmod 755 /usr/local/bin/rustup-init +ENV PATH="/root/.cargo/bin:${PATH}" + +# JAVA_VERSION and NODE_VERSION are declared only now, below the layers every tag +# shares: a build arg is part of the cache key of each RUN that follows its ARG, +# so declaring them at the top would rebuild (and re-hash) those layers per tag. +ARG JAVA_VERSION +ARG NODE_VERSION + +# https://adoptium.net/installation/linux +# add the gpg key and apt repo, then install the temurin jdk +RUN mkdir -p /etc/apt/keyrings \ + && wget -qO /etc/apt/keyrings/adoptium.asc https://packages.adoptium.net/artifactory/api/gpg/key/public \ + && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ + && apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends temurin-$JAVA_VERSION-jdk \ + && rm -rf /var/lib/apt/lists/* +ENV JAVA_HOME=/usr/lib/jvm/temurin-$JAVA_VERSION-jdk-$TARGETARCH + +# install nodejs from NodeSource. Replaces the base image's Ubuntu-default node +# (currently node 18 on noble) with a current LTS that user workflows can rely +# on without an extra setup-node step. +RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /actions-runner LABEL maintainer="asen.lekov@dodi.li" # Jobs run as root here, and GNU tar run as root keeps the owner ids stored in an @@ -102,9 +160,9 @@ LABEL maintainer="asen.lekov@dodi.li" ENV TAR_OPTIONS=--no-same-owner # Tool caches that outlive a job. Jobs run as root (HOME=/root), so these are -# the tools' own defaults made explicit: docker-compose.yml mounts one named -# volume on each of them. Mount the specific directories, never ~/.cache as a -# whole, or it would shadow the pnpm volume at ~/.cache/pnpm. +# the tools' own defaults made explicit: the preflight links each of them into +# the runner's slot (see preflight-entrypoint.sh). Link the specific +# directories, never ~/.cache as a whole, or it would shadow ~/.cache/pnpm. # - mise: tool installs (data) and downloads (cache). jdx/mise-action and # `mise install` fill these. # - hk: HK_CACHE_DIR is its config/tool cache. HK_PKL_CACHE_DIR defaults to @@ -115,37 +173,87 @@ ENV MISE_DATA_DIR=/root/.local/share/mise \ HK_CACHE_DIR=/root/.cache/hk \ HK_PKL_CACHE_DIR=/root/.cache/hk/pklr -# Job hooks: JOB_STARTED is the poison sweep (covers OOM / killed runner); -# JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches. -COPY pre-job.sh /usr/local/bin/pre-job.sh -COPY cleanup.sh /usr/local/bin/cleanup.sh -# pruner.sh is not a hook: the `pruner` compose service runs it nightly -# (LRU cache caps + docker prune), keeping those scans off the job path. -COPY pruner.sh /usr/local/bin/pruner.sh -RUN chmod +x /usr/local/bin/pre-job.sh /usr/local/bin/cleanup.sh /usr/local/bin/pruner.sh -ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh -ENV ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh - -# Preflight entrypoint: validates the persisted runner registration against -# GitHub before handing off to the upstream entrypoint. Upstream reuses -# CONFIGURED_ACTIONS_RUNNER_FILES_DIR credentials unconditionally, which -# crash-loops the container when the runner was removed server-side; the -# preflight wipes provably-dead registrations so a fresh ACCESS_TOKEN -# registration happens instead. See preflight-entrypoint.sh for details. -COPY preflight-entrypoint.sh /usr/local/bin/preflight-entrypoint.sh -RUN chmod +x /usr/local/bin/preflight-entrypoint.sh -# Image identity. deploy.yml passes the image tag and the commit it was built -# from; jobs read them (e.g. the runner report) to tell which image they ran -# on. They default to "dev" for local builds. Declared last: the values change -# on every build, so anything after them would never hit the layer cache. -ARG RUNNER_IMAGE_TAG=dev -ARG RUNNER_IMAGE_REVISION=dev -ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ - RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION +# Job hooks: JOB_STARTED marks the slot busy (and sweeps up after a killed +# job); JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches. +# The scripts themselves come from the `scripts` stage, copied last into each +# target (see there). +ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh \ + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh +# preflight-entrypoint.sh claims the runner's slot, links its caches, then +# validates a persisted registration before handing off to the upstream +# entrypoint (see the script for details). ENTRYPOINT ["/usr/local/bin/preflight-entrypoint.sh"] # Declaring ENTRYPOINT resets the CMD inherited from the base image, so the # base image's CMD is restated here. See # https://github.com/myoung34/docker-github-actions-runner#environment-variables # for how to use the image CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] + +################################################################################ +# scripts: this repo's runtime scripts, the files edited most often. Every +# target copies them as its LAST layer before the identity, so editing one +# never invalidates (or re-pushes) the SDK and NDK layers below. +# - pruner.sh is not a hook: the `pruner` compose service runs it (cache +# budgets + docker prune), keeping those scans off the job path +# - --chmod sets the mode in the COPY layer itself +################################################################################ +FROM scratch AS scripts +COPY --chmod=755 preflight-entrypoint.sh pre-job.sh cleanup.sh pruner.sh /usr/local/bin/ + +################################################################################ +# runner-slim (`--target runner-slim`): core without the Android SDK +# - for arm64 hosts (Android's build tools are x86_64-only), JS/JVM-only pools +# and small disks +################################################################################ +FROM core AS runner-slim +COPY --from=scripts / / +# Image identity. deploy.yml passes the image tag and the commit it was built +# from; jobs read them (e.g. the runner report) to tell which image they ran +# on. They default to "dev" for local builds. Declared last in every target +# (the same three lines close runner-ndk and runner): the values change on +# every build, so anything after them would never hit the layer cache. +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION + +################################################################################ +# android: core + the Android SDK, the base of runner and runner-ndk +# - the SDK sits on top of the JDK/Node layers: the weekly SDK bump then only +# replaces this layer, and hosts keep the JDK/Node layers they already have. +# Its blob is the same for every JDK/Node tag (the sdk stage never sees +# JAVA_VERSION/NODE_VERSION), so the registry stores and CI builds it once. +################################################################################ +FROM core AS android +ARG ANDROID_ROOT +COPY --from=sdk ${ANDROID_ROOT} ${ANDROID_ROOT} +# Set env variable for SDK Root (https://developer.android.com/studio/command-line/variables) +# ANDROID_HOME is deprecated, but older versions of Gradle rely on it +ENV ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk \ + ANDROID_HOME=$ANDROID_ROOT/sdk \ + PATH="${PATH}:${ANDROID_ROOT}/sdk/platform-tools/" + +################################################################################ +# runner-ndk (opt-in, `--target runner-ndk`): android + NDK + the SDK's cmake +# - the NDK and cmake;3.22.1 are several GB that most consumers never use +################################################################################ +FROM android AS runner-ndk +ARG ANDROID_ROOT +COPY --from=sdk-ndk /ndk-sdk/ndk ${ANDROID_ROOT}/sdk/ndk +COPY --from=sdk-ndk /ndk-sdk/cmake ${ANDROID_ROOT}/sdk/cmake +COPY --from=scripts / / +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION + +################################################################################ +# runner (default, MUST stay the last stage so a plain `docker build` makes it) +################################################################################ +FROM android AS runner +COPY --from=scripts / / +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION From 855ebb9d283acda495932c4610a1faebe3e877cc Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Mon, 19 Oct 2026 21:04:17 +0300 Subject: [PATCH 10/11] ci: LTS+latest JDK matrix, per-platform layer cache, slim/ndk images, App-token SDK update PRs - JDK 17, 21, 25, 27 x Node 22, 24: deploy 20 builds + 10 merges (was 32 + 16) incl. slim and ndk, PR 8 legs (was 11) - type=gha layer cache scoped per platform/variant, timeout-minutes, zstd-compressed layers (#15, #17) - slim and ndk tags; upstream's build-tools / NDK-version tags kept, NDK ones on the ndk variant only - tests/image-smoke.sh runs on the PR's amd64 std/slim/ndk legs; shellcheck covers every script; compose config validated - check-sdk-updates opens and auto-merges PRs with the org App token (APP_ID / APP_PRIVATE_KEY secrets) so CI runs (#17) - drops non-LTS JDKs once a newer GA lands Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 76 +++++++++++++---- .github/workflows/check-sdk-updates.yml | 99 +++++++++++++++-------- .github/workflows/deploy.yml | 103 ++++++++++++++++-------- matrix.json | 4 - tests/image-smoke.sh | 78 ++++++++++++++++++ 5 files changed, 272 insertions(+), 88 deletions(-) create mode 100755 tests/image-smoke.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d7f0f64..783e42b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,20 +13,42 @@ jobs: scripts: name: Shell scripts runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v7 - name: shellcheck - run: shellcheck -x pruner.sh tests/*.sh + run: | + set -euo pipefail + # nullglob: a pattern that matches nothing (e.g. hosts/*.sh) must + # drop out instead of being passed to shellcheck as a literal. + shopt -s nullglob + files=(*.sh hosts/*.sh tests/*.sh) + if [ "${#files[@]}" -eq 0 ]; then + echo "::error::No shell scripts found to check" + exit 1 + fi + shellcheck -x "${files[@]}" + + - name: docker compose config + run: | + set -euo pipefail + cp .env.example .env + docker compose config -q - name: tests run: | set -euo pipefail - for t in tests/*.sh; do bash "$t"; done + for t in tests/*.sh; do + # image-smoke.sh needs a built image; it runs in the build job. + [ "$(basename "$t")" = image-smoke.sh ] && continue + bash "$t" + done matrix: name: Generate build matrix runs-on: ubuntu-latest + timeout-minutes: 5 outputs: matrix: ${{ steps.generate.outputs.matrix }} steps: @@ -37,26 +59,35 @@ jobs: run: | set -euo pipefail # PR builds deliberately cover less than deploy.yml: the full - # java x node x arch product is ~28 images, which is far more than a - # dependency bump needs. Instead we cover every JDK against the - # default Node, every Node against the default JDK, and add a single - # arm64 build of the default pair so arch-specific breakage (e.g. the - # TARGETARCH-derived JAVA_HOME) is still caught before merge. + # java x node x arch product is far more than a dependency bump + # needs. Instead we cover every JDK against the default Node, every + # Node against the default JDK, and add a single arm64 build of the + # default pair so arch-specific breakage (e.g. the TARGETARCH-derived + # JAVA_HOME) is still caught before merge. The amd64 default pair + # also gets the image smoke test (smoke: true), and the opt-in + # runner-ndk and runner-slim targets get one amd64 build + smoke test + # each, since deploy.yml publishes them. MATRIX=$(jq -c ' .default_jdk as $dj | .default_node as $dn | ( [ .java_versions[] | {java_version: ., node_version: $dn} ] + [ .node_versions[] | {java_version: $dj, node_version: .} ] | unique ) as $combos - | { include: - ( [ $combos[] | . + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest"} ] - + [ {java_version: $dj, node_version: $dn, - platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm"} ] ) }' matrix.json) - echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + | def leg($arch; $runner): {platform: "linux/\($arch)", arch: $arch, runner: $runner}; + def std: {variant: "std", target: "runner"}; + { include: + ( [ $combos[] | . + std + leg("amd64"; "ubuntu-latest") + + {smoke: (.java_version == $dj and .node_version == $dn)} ] + + [ {java_version: $dj, node_version: $dn} + std + leg("arm64"; "ubuntu-24.04-arm") + {smoke: false} ] + + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "ndk", target: "runner-ndk"} ] + + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "slim", target: "runner-slim"} ] ) + | map(. + {cache_scope: (if .variant == "std" then .arch else "\(.variant)-\(.arch)" end)}) }' matrix.json) + echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.arch }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} + name: Build ${{ matrix.arch }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} (${{ matrix.variant }}) needs: matrix runs-on: ${{ matrix.runner }} + timeout-minutes: 45 strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} @@ -77,17 +108,29 @@ jobs: exit 1 fi - echo "runner_tag=${RUNNER_TAG}" >> $GITHUB_OUTPUT + echo "runner_tag=${RUNNER_TAG}" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # Layer cache lives in the GitHub Actions cache, one scope per platform (and + # per ndk/slim variant, matching deploy.yml) so + # the amd64 and arm64 legs never overwrite each other. PR builds read and + # write it too; branch scoping keeps a PR from polluting main's cache. + # `load` is only set on the smoke leg: the other legs have nothing to run. - name: Build (no push) uses: docker/build-push-action@v7 with: context: . + target: ${{ matrix.target }} platforms: ${{ matrix.platform }} push: false + load: ${{ matrix.smoke }} + tags: github-runner:smoke-${{ matrix.variant }} + cache-from: | + type=gha,scope=${{ matrix.arch }} + ${{ matrix.cache_scope != matrix.arch && format('type=gha,scope={0}', matrix.cache_scope) || '' }} + cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} build-args: | VERSION=${{ steps.versions.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} @@ -95,11 +138,16 @@ jobs: RUNNER_IMAGE_TAG=pr-${{ github.event.pull_request.number }} RUNNER_IMAGE_REVISION=${{ github.event.pull_request.head.sha }} + - name: Smoke test image + if: matrix.smoke + run: bash tests/image-smoke.sh github-runner:smoke-${{ matrix.variant }} "${{ matrix.java_version }}" "${{ matrix.node_version }}" "${{ matrix.variant }}" + gate: name: All builds passed needs: [scripts, matrix, build] if: always() runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Check that all required jobs succeeded run: | diff --git a/.github/workflows/check-sdk-updates.yml b/.github/workflows/check-sdk-updates.yml index ba0d8c6..94ae751 100644 --- a/.github/workflows/check-sdk-updates.yml +++ b/.github/workflows/check-sdk-updates.yml @@ -5,25 +5,41 @@ on: - cron: '0 9 * * 1' # Weekly on Monday at 9 AM UTC workflow_dispatch: +# Writes (branch push, PR, auto-merge) go through the GitHub App token minted +# below, not GITHUB_TOKEN: PRs opened with GITHUB_TOKEN never trigger +# build.yml, so their required checks would never report. permissions: - contents: write - pull-requests: write + contents: read jobs: check-updates: name: Check for updates runs-on: ubuntu-latest + timeout-minutes: 15 steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-contents: write + permission-pull-requests: write + - uses: actions/checkout@v7 + with: + token: ${{ steps.app-token.outputs.token }} - name: Check for new JDK versions id: jdk run: | - # Get available Temurin releases >= 17 + # Get available Temurin GA releases >= 17 RESPONSE=$(curl -sf https://api.adoptium.net/v3/info/available_releases || echo "") if [ -z "$RESPONSE" ]; then echo "Failed to fetch Adoptium releases API" - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" exit 0 fi @@ -41,12 +57,12 @@ jobs: fi done - echo "new_versions=${NEW_VERSIONS}" >> $GITHUB_OUTPUT + echo "new_versions=${NEW_VERSIONS}" >> "$GITHUB_OUTPUT" if [ -n "$NEW_VERSIONS" ]; then - echo "has_new=true" >> $GITHUB_OUTPUT + echo "has_new=true" >> "$GITHUB_OUTPUT" echo "Found new JDK versions: $NEW_VERSIONS" else - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" echo "No new JDK versions found" fi @@ -61,7 +77,7 @@ jobs: # which falsely includes long-EOL majors (10, 12, 14, 16, 18). if ! SCHEDULE=$(curl -sf https://raw.githubusercontent.com/nodejs/Release/main/schedule.json); then echo "Failed to fetch Node release schedule" - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" exit 0 fi @@ -88,12 +104,12 @@ jobs: fi done - echo "new_versions=${NEW_VERSIONS}" >> $GITHUB_OUTPUT + echo "new_versions=${NEW_VERSIONS}" >> "$GITHUB_OUTPUT" if [ -n "$NEW_VERSIONS" ]; then - echo "has_new=true" >> $GITHUB_OUTPUT + echo "has_new=true" >> "$GITHUB_OUTPUT" echo "Found new Node LTS versions: $NEW_VERSIONS" else - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" echo "No new Node LTS versions found" fi @@ -107,12 +123,12 @@ jobs: # are listed too), which then break the unattended Docker build when # sdkmanager demands the separate Android SDK Preview license. The manifest # carries reliable preview markers, so we parse it with a real XML parser - # and skip any package that is obsolete, carries a non-empty , or - # uses the android-sdk-preview-license. Stable platforms since android-37.0 - # carry an EMPTY , so only a non-empty one marks a preview. + # and skip any package that is obsolete, carries a non-empty + # (stable platforms have an empty one), or uses the + # android-sdk-preview-license. if ! wget -q "https://dl.google.com/android/repository/repository2-3.xml" -O /tmp/android-repo.xml; then echo "Failed to download SDK repository manifest" - echo "updated=false" >> $GITHUB_OUTPUT + echo "updated=false" >> "$GITHUB_OUTPUT" exit 0 fi if ! SDK_VERSIONS=$(python3 - /tmp/android-repo.xml <<'PY' @@ -127,6 +143,9 @@ jobs: if pkg.get('obsolete') == 'true': return True for e in pkg.iter(): + # Stable platforms carry an *empty* + # (e.g. android-37.0/37.1/37.2); previews name the next + # release there (DEV, CinnamonBun, ...). if local(e.tag) == 'codename' and (e.text or '').strip(): return True if local(e.tag) == 'uses-license' and e.get('ref') == 'android-sdk-preview-license': @@ -160,7 +179,7 @@ jobs: PY ); then echo "Failed to parse SDK repository manifest" - echo "updated=false" >> $GITHUB_OUTPUT + echo "updated=false" >> "$GITHUB_OUTPUT" exit 0 fi eval "$SDK_VERSIONS" @@ -170,12 +189,14 @@ jobs: CURRENT_BUILD_TOOLS=$(grep '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) CURRENT_NDK=$(grep '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - echo "latest_platform=${LATEST_PLATFORM}" >> $GITHUB_OUTPUT - echo "latest_build_tools=${LATEST_BUILD_TOOLS}" >> $GITHUB_OUTPUT - echo "latest_ndk=${LATEST_NDK}" >> $GITHUB_OUTPUT - echo "current_platform=${CURRENT_PLATFORM}" >> $GITHUB_OUTPUT - echo "current_build_tools=${CURRENT_BUILD_TOOLS}" >> $GITHUB_OUTPUT - echo "current_ndk=${CURRENT_NDK}" >> $GITHUB_OUTPUT + { + echo "latest_platform=${LATEST_PLATFORM}" + echo "latest_build_tools=${LATEST_BUILD_TOOLS}" + echo "latest_ndk=${LATEST_NDK}" + echo "current_platform=${CURRENT_PLATFORM}" + echo "current_build_tools=${CURRENT_BUILD_TOOLS}" + echo "current_ndk=${CURRENT_NDK}" + } >> "$GITHUB_OUTPUT" # Check if updates are needed UPDATED=false @@ -192,7 +213,7 @@ jobs: UPDATED=true fi - echo "updated=${UPDATED}" >> $GITHUB_OUTPUT + echo "updated=${UPDATED}" >> "$GITHUB_OUTPUT" echo "SDK check results:" echo " Platform: ${CURRENT_PLATFORM} -> ${LATEST_PLATFORM}" echo " Build tools: ${CURRENT_BUILD_TOOLS} -> ${LATEST_BUILD_TOOLS}" @@ -200,13 +221,23 @@ jobs: - name: Apply JDK updates if: steps.jdk.outputs.has_new == 'true' + env: + NEW_VERSIONS: ${{ steps.jdk.outputs.new_versions }} run: | - for v in ${{ steps.jdk.outputs.new_versions }}; do - jq --arg v "$v" '.java_versions += [$v] | .java_versions = (.java_versions | sort_by(tonumber))' matrix.json > tmp.json && mv tmp.json matrix.json + for v in $NEW_VERSIONS; do + jq --arg v "$v" '.java_versions += [$v]' matrix.json > tmp.json && mv tmp.json matrix.json done - # Update default to highest version - jq '.default_jdk = (.java_versions | sort_by(tonumber) | last)' matrix.json > tmp.json && mv tmp.json matrix.json + # Retention policy: keep the LTS lines (17, 21, 25, then one every + # two years: 29, 33, ...) plus the newest GA release, and drop any + # other non-LTS version once it is no longer the newest. The default + # is always the newest. + jq ' + def lts: . >= 17 and ((. - 17) % 4 == 0); + ([.java_versions[] | tonumber] | unique) as $all + | ($all | max) as $newest + | .java_versions = [ $all[] | select(. == $newest or lts) | tostring ] + | .default_jdk = (.java_versions | last)' matrix.json > tmp.json && mv tmp.json matrix.json echo "Updated matrix.json:" cat matrix.json @@ -239,7 +270,7 @@ jobs: - name: Create pull request if: steps.jdk.outputs.has_new == 'true' || steps.sdk.outputs.updated == 'true' || steps.node.outputs.has_new == 'true' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | BRANCH="automated/sdk-jdk-updates" @@ -261,7 +292,7 @@ jobs: MSG="$MSG SDK" fi MSG="$MSG versions" - git commit -m "$MSG" + git commit -m "chore(deps): $MSG" git push -f origin "$BRANCH" # Build PR body @@ -302,15 +333,13 @@ jobs: EXISTING=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null || true) if [ -z "$EXISTING" ]; then gh pr create \ - --title "Update SDK/JDK/Node versions" \ + --title "chore(deps): update SDK/JDK/Node versions" \ --body-file /tmp/pr-body.txt \ --head "$BRANCH" else echo "PR #${EXISTING} already exists, updated with force push" fi - # No auto-merge on purpose: this PR is opened with GITHUB_TOKEN, and GitHub - # does not trigger workflows for PRs created that way. build.yml never runs, - # so the "All builds passed" check never reports and an auto-merge request - # would hang forever. Merge manually (which runs the builds), or add a - # fine-grained PAT and restore upstream's auto-merge step. + # The PR is opened with the App token, so build.yml runs and the + # "All builds passed" check reports; auto-merge waits for it. + gh pr merge --auto --squash "$BRANCH" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 53aa0ec..caaf3de 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,6 +12,7 @@ jobs: matrix: name: Generate build matrix runs-on: ubuntu-latest + timeout-minutes: 5 outputs: build_matrix: ${{ steps.generate.outputs.build_matrix }} merge_matrix: ${{ steps.generate.outputs.merge_matrix }} @@ -22,24 +23,37 @@ jobs: id: generate run: | set -euo pipefail + # Every JDK x Node pair is published as the `runner` target (variant + # "std"). The default pair is additionally published as `runner-ndk` + # (the image plus the NDK) and `runner-slim` (no Android SDK). Each + # variant has its own digests, cache scope and tags so they never mix + # with the standard image. ALL=$(jq -c ' .default_jdk as $dj | .default_node as $dn | [ .java_versions[] as $j | .node_versions[] as $n | { java_version: $j, node_version: $n, is_default_jdk: ($j == $dj), - is_default_node: ($n == $dn) } ] as $combos + is_default_node: ($n == $dn), + variant: "std", target: "runner" } ] as $std + | ( [ $std[] | select(.is_default_jdk and .is_default_node) ] ) as $default_pair + | ( [ $default_pair[] | . + {variant: "ndk", target: "runner-ndk"} ] + + [ $default_pair[] | . + {variant: "slim", target: "runner-slim"} ] ) as $extra + | ( $std + $extra ) as $combos | { build: { include: [ $combos[] as $c | - ($c + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest"}), - ($c + {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm"}) ] }, + ($c + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest", + cache_scope: (if $c.variant == "std" then "amd64" else "\($c.variant)-amd64" end)}), + ($c + {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm", + cache_scope: (if $c.variant == "std" then "arm64" else "\($c.variant)-arm64" end)}) ] }, merge: { include: $combos } }' matrix.json) - echo "build_matrix=$(jq -c .build <<< "$ALL")" >> $GITHUB_OUTPUT - echo "merge_matrix=$(jq -c .merge <<< "$ALL")" >> $GITHUB_OUTPUT + echo "build_matrix=$(jq -c .build <<< "$ALL")" >> "$GITHUB_OUTPUT" + echo "merge_matrix=$(jq -c .merge <<< "$ALL")" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.platform }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} + name: Build ${{ matrix.platform }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} (${{ matrix.variant }}) needs: matrix runs-on: ${{ matrix.runner }} + timeout-minutes: 45 strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.build_matrix) }} @@ -53,19 +67,18 @@ jobs: RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) - BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) - NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then + if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ]; then echo "::error::Failed to extract versions from Dockerfile" exit 1 fi - echo "runner_tag=${RUNNER_TAG}" >> $GITHUB_OUTPUT - echo "runner_version=${RUNNER_VERSION}" >> $GITHUB_OUTPUT - echo "compile_sdk=${COMPILE_SDK}" >> $GITHUB_OUTPUT - echo "build_tools=${BUILD_TOOLS}" >> $GITHUB_OUTPUT - echo "ndk_version=${NDK_VERSION}" >> $GITHUB_OUTPUT + # build_tools / ndk_version are only needed for the merge job's tags. + { + echo "runner_tag=${RUNNER_TAG}" + echo "runner_version=${RUNNER_VERSION}" + echo "compile_sdk=${COMPILE_SDK}" + } >> "$GITHUB_OUTPUT" - name: Login to GitHub Container Registry uses: docker/login-action@v4 @@ -86,14 +99,22 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/github-runner tags: | - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}${{ matrix.variant == 'ndk' && '-ndk' || '' }},enable=${{ matrix.variant != 'slim' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} - name: Build and push by digest id: build uses: docker/build-push-action@v7 with: context: . + target: ${{ matrix.target }} platforms: ${{ matrix.platform }} + # Layer cache in the GitHub Actions cache, one scope per platform + # (the ndk and slim variants keep their own scope and read the standard one). + cache-from: | + type=gha,scope=${{ matrix.arch }} + ${{ matrix.cache_scope != matrix.arch && format('type=gha,scope={0}', matrix.cache_scope) || '' }} + cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} # Disable SLSA provenance: it wraps the image in a per-registry # attestation index which produces unknown/unknown manifest entries # and an extra attestation manifest per arch. Nothing in this @@ -106,7 +127,10 @@ jobs: NODE_VERSION=${{ matrix.node_version }} RUNNER_IMAGE_TAG=${{ steps.meta.outputs.version }} RUNNER_IMAGE_REVISION=${{ github.sha }} - outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/github-runner,push-by-digest=true,name-canonical=true,push=true + # zstd layers pull and decompress faster on small hosts (Raspberry Pi) than gzip. + # force-compression recompresses cached/base layers too; oci-mediatypes is + # required for zstd. `imagetools create` below merges the per-arch OCI manifests. + outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/github-runner,push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3,force-compression=true,oci-mediatypes=true - name: Export digest run: | @@ -118,15 +142,16 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v7 with: - name: digests-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-${{ matrix.arch }} + name: digests-${{ matrix.variant }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-${{ matrix.arch }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 merge: - name: Merge JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} + name: Merge JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} (${{ matrix.variant }}) needs: [matrix, build] runs-on: ubuntu-latest + timeout-minutes: 15 strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.merge_matrix) }} @@ -148,15 +173,17 @@ jobs: exit 1 fi - echo "runner_version=${RUNNER_VERSION}" >> $GITHUB_OUTPUT - echo "compile_sdk=${COMPILE_SDK}" >> $GITHUB_OUTPUT - echo "build_tools=${BUILD_TOOLS}" >> $GITHUB_OUTPUT - echo "ndk_version=${NDK_VERSION}" >> $GITHUB_OUTPUT + { + echo "runner_version=${RUNNER_VERSION}" + echo "compile_sdk=${COMPILE_SDK}" + echo "build_tools=${BUILD_TOOLS}" + echo "ndk_version=${NDK_VERSION}" + } >> "$GITHUB_OUTPUT" - name: Download digests uses: actions/download-artifact@v8 with: - pattern: digests-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-* + pattern: digests-${{ matrix.variant }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-* merge-multiple: true path: ${{ runner.temp }}/digests @@ -176,18 +203,22 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/github-runner tags: | - type=raw,value=jdk${{ matrix.java_version }},enable=${{ matrix.is_default_node }} - type=raw,value=node${{ matrix.node_version }},enable=${{ matrix.is_default_jdk }} - type=raw,value=jdk${{ matrix.java_version }}-node${{ matrix.node_version }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }} - # Same image, with the NDK it ships in the tag: consumers that pin an NDK (Android - # Gradle ndkVersion) can let Renovate derive it from this tag so image and NDK move together. - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }} - # Same image, with every Gradle-pinnable SDK component in the tag (compileSdk, build-tools, - # NDK), so consumers can let Renovate derive all three from one tag. The -sdk-ndk tag above - # stays for consumers already parsing it. - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }} - type=raw,value=latest,enable=${{ matrix.is_default_jdk && matrix.is_default_node }} + type=raw,value=jdk${{ matrix.java_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_node }} + type=raw,value=node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_jdk }} + type=raw,value=jdk${{ matrix.java_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} + # Every Gradle-pinnable SDK component in the tag (compileSdk, build-tools), so + # consumers can let Renovate derive them from the image pin. + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} + type=raw,value=latest,enable=${{ matrix.variant == 'std' && matrix.is_default_jdk && matrix.is_default_node }} + type=raw,value=ndk,enable=${{ matrix.variant == 'ndk' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}-ndk,enable=${{ matrix.variant == 'ndk' }} + # Only the ndk variant ships an NDK, so only it spells the NDK version out + # (upstream's -sdk-ndk- and -sdk-bt-ndk- tags; Gradle ndkVersion via Renovate). + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} + type=raw,value=slim,enable=${{ matrix.variant == 'slim' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests @@ -198,6 +229,8 @@ jobs: TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") # Source: one per arch, by digest, from the GHCR push in the build job. SOURCES=$(printf "${IMAGE}@sha256:%s " *) + # TAGS and SOURCES are deliberately word-split into arguments. + # shellcheck disable=SC2086 docker buildx imagetools create $TAGS $SOURCES for tag in $(jq -cr '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON"); do diff --git a/matrix.json b/matrix.json index bf1fe20..3b68981 100644 --- a/matrix.json +++ b/matrix.json @@ -2,11 +2,7 @@ "java_versions": [ "17", "21", - "22", - "23", - "24", "25", - "26", "27" ], "default_jdk": "27", diff --git a/tests/image-smoke.sh b/tests/image-smoke.sh new file mode 100755 index 0000000..d00ee6d --- /dev/null +++ b/tests/image-smoke.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Smoke test for a built runner image: the toolchain a job relies on must be +# present and runnable, with the entrypoint bypassed so no runner registers. +# bash tests/image-smoke.sh [expected-jdk-major] [expected-node-major] [std|ndk|slim] +# Variants: std (runner) and ndk (runner-ndk) must carry the Android SDK; slim +# (runner-slim) must carry java/node/tools but no Android SDK. +# The check snippets are single-quoted on purpose: they expand inside the image. +# shellcheck disable=SC2016 +set -euo pipefail + +image="${1:?usage: tests/image-smoke.sh [jdk-major] [node-major] [std|ndk|slim]}" +want_jdk="${2:-}" +want_node="${3:-}" +variant="${4:-std}" +case "$variant" in + std | ndk | slim) ;; + *) echo "unknown variant '$variant' (want std, ndk or slim)" >&2; exit 2 ;; +esac + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } +pass() { echo "ok: $*"; } +indent() { printf ' %s\n' "${1//$'\n'/$'\n '}"; } + +# in_image