diff --git a/.env.example b/.env.example index c3d0116..03d57fc 100644 --- a/.env.example +++ b/.env.example @@ -6,22 +6,31 @@ APP_PRIVATE_KEY= # Organization the runners register to APP_LOGIN= +# How many runners this host runs; `hosts/sizing.sh [light|large]` suggests a +# value from its CPUs, memory and disk. +RUNNERS=3 +# Host directory for every runner's work dir and caches, mounted at the same +# path in the containers (slots//work, slots//cache). Put it on a disk +# with room; old named volumes from before slots can be removed. +# RUNNER_DATA_DIR=/var/lib/github-runner + # Runner identity. The runner adds self-hosted, the OS and the architecture to # LABELS itself; add the capability labels your workflows select on. RUNNER_NAME_PREFIX=runner LABELS=docker RUNNER_GROUP=Default # One runner can join an extra pool, e.g. a host whose runners are `light` with -# one that also builds: +# one that also builds (slot 3): # RUNNER_3_LABELS=docker,x86,light,large -# Coordination runners (two, labelled `control`), for the short jobs a merge -# waits on. Off unless the profile is set: -# COMPOSE_PROFILES=control +# Coordination runners labelled CONTROL_LABELS, for the short jobs a merge +# waits on. None unless set: +# CONTROL_RUNNERS=2 # CONTROL_LABELS=control -# Small hosts without room for per-runner caches (e.g. ARM64 boards): -# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml +# Small hosts without room for caches (e.g. ARM64 boards): jobs then start +# cold. The pruner's budget already shrinks with the disk, so try that first. +# RUNNER_CACHES=off # Action archive cache (see README). Space-separated owner/repo@ref lists. # PREFETCH_SOURCES=my-org/my-workflows@main # repos scanned for `uses:` @@ -29,7 +38,7 @@ RUNNER_GROUP=Default # Other tunables (defaults shown) # RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner -# CACHE_CAP_BYTES=21474836480 +# CACHE_BUDGET_BYTES=64424509440 # host-wide cache budget, split across slots # PRUNE_INTERVAL_SECONDS=86400 # PREFETCH_INTERVAL_SECONDS=3600 # Any non-empty value disables runner self-update ("false" too). diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d7f0f64..783e42b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,20 +13,42 @@ jobs: scripts: name: Shell scripts runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v7 - name: shellcheck - run: shellcheck -x pruner.sh tests/*.sh + run: | + set -euo pipefail + # nullglob: a pattern that matches nothing (e.g. hosts/*.sh) must + # drop out instead of being passed to shellcheck as a literal. + shopt -s nullglob + files=(*.sh hosts/*.sh tests/*.sh) + if [ "${#files[@]}" -eq 0 ]; then + echo "::error::No shell scripts found to check" + exit 1 + fi + shellcheck -x "${files[@]}" + + - name: docker compose config + run: | + set -euo pipefail + cp .env.example .env + docker compose config -q - name: tests run: | set -euo pipefail - for t in tests/*.sh; do bash "$t"; done + for t in tests/*.sh; do + # image-smoke.sh needs a built image; it runs in the build job. + [ "$(basename "$t")" = image-smoke.sh ] && continue + bash "$t" + done matrix: name: Generate build matrix runs-on: ubuntu-latest + timeout-minutes: 5 outputs: matrix: ${{ steps.generate.outputs.matrix }} steps: @@ -37,26 +59,35 @@ jobs: run: | set -euo pipefail # PR builds deliberately cover less than deploy.yml: the full - # java x node x arch product is ~28 images, which is far more than a - # dependency bump needs. Instead we cover every JDK against the - # default Node, every Node against the default JDK, and add a single - # arm64 build of the default pair so arch-specific breakage (e.g. the - # TARGETARCH-derived JAVA_HOME) is still caught before merge. + # java x node x arch product is far more than a dependency bump + # needs. Instead we cover every JDK against the default Node, every + # Node against the default JDK, and add a single arm64 build of the + # default pair so arch-specific breakage (e.g. the TARGETARCH-derived + # JAVA_HOME) is still caught before merge. The amd64 default pair + # also gets the image smoke test (smoke: true), and the opt-in + # runner-ndk and runner-slim targets get one amd64 build + smoke test + # each, since deploy.yml publishes them. MATRIX=$(jq -c ' .default_jdk as $dj | .default_node as $dn | ( [ .java_versions[] | {java_version: ., node_version: $dn} ] + [ .node_versions[] | {java_version: $dj, node_version: .} ] | unique ) as $combos - | { include: - ( [ $combos[] | . + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest"} ] - + [ {java_version: $dj, node_version: $dn, - platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm"} ] ) }' matrix.json) - echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + | def leg($arch; $runner): {platform: "linux/\($arch)", arch: $arch, runner: $runner}; + def std: {variant: "std", target: "runner"}; + { include: + ( [ $combos[] | . + std + leg("amd64"; "ubuntu-latest") + + {smoke: (.java_version == $dj and .node_version == $dn)} ] + + [ {java_version: $dj, node_version: $dn} + std + leg("arm64"; "ubuntu-24.04-arm") + {smoke: false} ] + + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "ndk", target: "runner-ndk"} ] + + [ {java_version: $dj, node_version: $dn} + leg("amd64"; "ubuntu-latest") + {smoke: true, variant: "slim", target: "runner-slim"} ] ) + | map(. + {cache_scope: (if .variant == "std" then .arch else "\(.variant)-\(.arch)" end)}) }' matrix.json) + echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.arch }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} + name: Build ${{ matrix.arch }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} (${{ matrix.variant }}) needs: matrix runs-on: ${{ matrix.runner }} + timeout-minutes: 45 strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} @@ -77,17 +108,29 @@ jobs: exit 1 fi - echo "runner_tag=${RUNNER_TAG}" >> $GITHUB_OUTPUT + echo "runner_tag=${RUNNER_TAG}" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # Layer cache lives in the GitHub Actions cache, one scope per platform (and + # per ndk/slim variant, matching deploy.yml) so + # the amd64 and arm64 legs never overwrite each other. PR builds read and + # write it too; branch scoping keeps a PR from polluting main's cache. + # `load` is only set on the smoke leg: the other legs have nothing to run. - name: Build (no push) uses: docker/build-push-action@v7 with: context: . + target: ${{ matrix.target }} platforms: ${{ matrix.platform }} push: false + load: ${{ matrix.smoke }} + tags: github-runner:smoke-${{ matrix.variant }} + cache-from: | + type=gha,scope=${{ matrix.arch }} + ${{ matrix.cache_scope != matrix.arch && format('type=gha,scope={0}', matrix.cache_scope) || '' }} + cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} build-args: | VERSION=${{ steps.versions.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} @@ -95,11 +138,16 @@ jobs: RUNNER_IMAGE_TAG=pr-${{ github.event.pull_request.number }} RUNNER_IMAGE_REVISION=${{ github.event.pull_request.head.sha }} + - name: Smoke test image + if: matrix.smoke + run: bash tests/image-smoke.sh github-runner:smoke-${{ matrix.variant }} "${{ matrix.java_version }}" "${{ matrix.node_version }}" "${{ matrix.variant }}" + gate: name: All builds passed needs: [scripts, matrix, build] if: always() runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Check that all required jobs succeeded run: | diff --git a/.github/workflows/check-sdk-updates.yml b/.github/workflows/check-sdk-updates.yml index fd3826b..94ae751 100644 --- a/.github/workflows/check-sdk-updates.yml +++ b/.github/workflows/check-sdk-updates.yml @@ -5,25 +5,41 @@ on: - cron: '0 9 * * 1' # Weekly on Monday at 9 AM UTC workflow_dispatch: +# Writes (branch push, PR, auto-merge) go through the GitHub App token minted +# below, not GITHUB_TOKEN: PRs opened with GITHUB_TOKEN never trigger +# build.yml, so their required checks would never report. permissions: - contents: write - pull-requests: write + contents: read jobs: check-updates: name: Check for updates runs-on: ubuntu-latest + timeout-minutes: 15 steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-contents: write + permission-pull-requests: write + - uses: actions/checkout@v7 + with: + token: ${{ steps.app-token.outputs.token }} - name: Check for new JDK versions id: jdk run: | - # Get available Temurin releases >= 17 + # Get available Temurin GA releases >= 17 RESPONSE=$(curl -sf https://api.adoptium.net/v3/info/available_releases || echo "") if [ -z "$RESPONSE" ]; then echo "Failed to fetch Adoptium releases API" - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" exit 0 fi @@ -41,12 +57,12 @@ jobs: fi done - echo "new_versions=${NEW_VERSIONS}" >> $GITHUB_OUTPUT + echo "new_versions=${NEW_VERSIONS}" >> "$GITHUB_OUTPUT" if [ -n "$NEW_VERSIONS" ]; then - echo "has_new=true" >> $GITHUB_OUTPUT + echo "has_new=true" >> "$GITHUB_OUTPUT" echo "Found new JDK versions: $NEW_VERSIONS" else - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" echo "No new JDK versions found" fi @@ -61,7 +77,7 @@ jobs: # which falsely includes long-EOL majors (10, 12, 14, 16, 18). if ! SCHEDULE=$(curl -sf https://raw.githubusercontent.com/nodejs/Release/main/schedule.json); then echo "Failed to fetch Node release schedule" - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" exit 0 fi @@ -88,12 +104,12 @@ jobs: fi done - echo "new_versions=${NEW_VERSIONS}" >> $GITHUB_OUTPUT + echo "new_versions=${NEW_VERSIONS}" >> "$GITHUB_OUTPUT" if [ -n "$NEW_VERSIONS" ]; then - echo "has_new=true" >> $GITHUB_OUTPUT + echo "has_new=true" >> "$GITHUB_OUTPUT" echo "Found new Node LTS versions: $NEW_VERSIONS" else - echo "has_new=false" >> $GITHUB_OUTPUT + echo "has_new=false" >> "$GITHUB_OUTPUT" echo "No new Node LTS versions found" fi @@ -107,11 +123,12 @@ jobs: # are listed too), which then break the unattended Docker build when # sdkmanager demands the separate Android SDK Preview license. The manifest # carries reliable preview markers, so we parse it with a real XML parser - # and skip any package that is obsolete, carries a element, or - # uses the android-sdk-preview-license. + # and skip any package that is obsolete, carries a non-empty + # (stable platforms have an empty one), or uses the + # android-sdk-preview-license. if ! wget -q "https://dl.google.com/android/repository/repository2-3.xml" -O /tmp/android-repo.xml; then echo "Failed to download SDK repository manifest" - echo "updated=false" >> $GITHUB_OUTPUT + echo "updated=false" >> "$GITHUB_OUTPUT" exit 0 fi if ! SDK_VERSIONS=$(python3 - /tmp/android-repo.xml <<'PY' @@ -126,7 +143,10 @@ jobs: if pkg.get('obsolete') == 'true': return True for e in pkg.iter(): - if local(e.tag) == 'codename': + # Stable platforms carry an *empty* + # (e.g. android-37.0/37.1/37.2); previews name the next + # release there (DEV, CinnamonBun, ...). + if local(e.tag) == 'codename' and (e.text or '').strip(): return True if local(e.tag) == 'uses-license' and e.get('ref') == 'android-sdk-preview-license': return True @@ -159,7 +179,7 @@ jobs: PY ); then echo "Failed to parse SDK repository manifest" - echo "updated=false" >> $GITHUB_OUTPUT + echo "updated=false" >> "$GITHUB_OUTPUT" exit 0 fi eval "$SDK_VERSIONS" @@ -169,12 +189,14 @@ jobs: CURRENT_BUILD_TOOLS=$(grep '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) CURRENT_NDK=$(grep '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - echo "latest_platform=${LATEST_PLATFORM}" >> $GITHUB_OUTPUT - echo "latest_build_tools=${LATEST_BUILD_TOOLS}" >> $GITHUB_OUTPUT - echo "latest_ndk=${LATEST_NDK}" >> $GITHUB_OUTPUT - echo "current_platform=${CURRENT_PLATFORM}" >> $GITHUB_OUTPUT - echo "current_build_tools=${CURRENT_BUILD_TOOLS}" >> $GITHUB_OUTPUT - echo "current_ndk=${CURRENT_NDK}" >> $GITHUB_OUTPUT + { + echo "latest_platform=${LATEST_PLATFORM}" + echo "latest_build_tools=${LATEST_BUILD_TOOLS}" + echo "latest_ndk=${LATEST_NDK}" + echo "current_platform=${CURRENT_PLATFORM}" + echo "current_build_tools=${CURRENT_BUILD_TOOLS}" + echo "current_ndk=${CURRENT_NDK}" + } >> "$GITHUB_OUTPUT" # Check if updates are needed UPDATED=false @@ -191,7 +213,7 @@ jobs: UPDATED=true fi - echo "updated=${UPDATED}" >> $GITHUB_OUTPUT + echo "updated=${UPDATED}" >> "$GITHUB_OUTPUT" echo "SDK check results:" echo " Platform: ${CURRENT_PLATFORM} -> ${LATEST_PLATFORM}" echo " Build tools: ${CURRENT_BUILD_TOOLS} -> ${LATEST_BUILD_TOOLS}" @@ -199,13 +221,23 @@ jobs: - name: Apply JDK updates if: steps.jdk.outputs.has_new == 'true' + env: + NEW_VERSIONS: ${{ steps.jdk.outputs.new_versions }} run: | - for v in ${{ steps.jdk.outputs.new_versions }}; do - jq --arg v "$v" '.java_versions += [$v] | .java_versions = (.java_versions | sort_by(tonumber))' matrix.json > tmp.json && mv tmp.json matrix.json + for v in $NEW_VERSIONS; do + jq --arg v "$v" '.java_versions += [$v]' matrix.json > tmp.json && mv tmp.json matrix.json done - # Update default to highest version - jq '.default_jdk = (.java_versions | sort_by(tonumber) | last)' matrix.json > tmp.json && mv tmp.json matrix.json + # Retention policy: keep the LTS lines (17, 21, 25, then one every + # two years: 29, 33, ...) plus the newest GA release, and drop any + # other non-LTS version once it is no longer the newest. The default + # is always the newest. + jq ' + def lts: . >= 17 and ((. - 17) % 4 == 0); + ([.java_versions[] | tonumber] | unique) as $all + | ($all | max) as $newest + | .java_versions = [ $all[] | select(. == $newest or lts) | tostring ] + | .default_jdk = (.java_versions | last)' matrix.json > tmp.json && mv tmp.json matrix.json echo "Updated matrix.json:" cat matrix.json @@ -238,7 +270,7 @@ jobs: - name: Create pull request if: steps.jdk.outputs.has_new == 'true' || steps.sdk.outputs.updated == 'true' || steps.node.outputs.has_new == 'true' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | BRANCH="automated/sdk-jdk-updates" @@ -260,7 +292,7 @@ jobs: MSG="$MSG SDK" fi MSG="$MSG versions" - git commit -m "$MSG" + git commit -m "chore(deps): $MSG" git push -f origin "$BRANCH" # Build PR body @@ -301,15 +333,13 @@ jobs: EXISTING=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null || true) if [ -z "$EXISTING" ]; then gh pr create \ - --title "Update SDK/JDK/Node versions" \ + --title "chore(deps): update SDK/JDK/Node versions" \ --body-file /tmp/pr-body.txt \ --head "$BRANCH" else echo "PR #${EXISTING} already exists, updated with force push" fi - # No auto-merge on purpose: this PR is opened with GITHUB_TOKEN, and GitHub - # does not trigger workflows for PRs created that way. build.yml never runs, - # so the "All builds passed" check never reports and an auto-merge request - # would hang forever. Merge manually (which runs the builds), or add a - # fine-grained PAT and restore upstream's auto-merge step. + # The PR is opened with the App token, so build.yml runs and the + # "All builds passed" check reports; auto-merge waits for it. + gh pr merge --auto --squash "$BRANCH" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 18ee23c..caaf3de 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,6 +12,7 @@ jobs: matrix: name: Generate build matrix runs-on: ubuntu-latest + timeout-minutes: 5 outputs: build_matrix: ${{ steps.generate.outputs.build_matrix }} merge_matrix: ${{ steps.generate.outputs.merge_matrix }} @@ -22,24 +23,37 @@ jobs: id: generate run: | set -euo pipefail + # Every JDK x Node pair is published as the `runner` target (variant + # "std"). The default pair is additionally published as `runner-ndk` + # (the image plus the NDK) and `runner-slim` (no Android SDK). Each + # variant has its own digests, cache scope and tags so they never mix + # with the standard image. ALL=$(jq -c ' .default_jdk as $dj | .default_node as $dn | [ .java_versions[] as $j | .node_versions[] as $n | { java_version: $j, node_version: $n, is_default_jdk: ($j == $dj), - is_default_node: ($n == $dn) } ] as $combos + is_default_node: ($n == $dn), + variant: "std", target: "runner" } ] as $std + | ( [ $std[] | select(.is_default_jdk and .is_default_node) ] ) as $default_pair + | ( [ $default_pair[] | . + {variant: "ndk", target: "runner-ndk"} ] + + [ $default_pair[] | . + {variant: "slim", target: "runner-slim"} ] ) as $extra + | ( $std + $extra ) as $combos | { build: { include: [ $combos[] as $c | - ($c + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest"}), - ($c + {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm"}) ] }, + ($c + {platform: "linux/amd64", arch: "amd64", runner: "ubuntu-latest", + cache_scope: (if $c.variant == "std" then "amd64" else "\($c.variant)-amd64" end)}), + ($c + {platform: "linux/arm64", arch: "arm64", runner: "ubuntu-24.04-arm", + cache_scope: (if $c.variant == "std" then "arm64" else "\($c.variant)-arm64" end)}) ] }, merge: { include: $combos } }' matrix.json) - echo "build_matrix=$(jq -c .build <<< "$ALL")" >> $GITHUB_OUTPUT - echo "merge_matrix=$(jq -c .merge <<< "$ALL")" >> $GITHUB_OUTPUT + echo "build_matrix=$(jq -c .build <<< "$ALL")" >> "$GITHUB_OUTPUT" + echo "merge_matrix=$(jq -c .merge <<< "$ALL")" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.platform }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} + name: Build ${{ matrix.platform }} JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} (${{ matrix.variant }}) needs: matrix runs-on: ${{ matrix.runner }} + timeout-minutes: 45 strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.build_matrix) }} @@ -59,9 +73,12 @@ jobs: exit 1 fi - echo "runner_tag=${RUNNER_TAG}" >> $GITHUB_OUTPUT - echo "runner_version=${RUNNER_VERSION}" >> $GITHUB_OUTPUT - echo "compile_sdk=${COMPILE_SDK}" >> $GITHUB_OUTPUT + # build_tools / ndk_version are only needed for the merge job's tags. + { + echo "runner_tag=${RUNNER_TAG}" + echo "runner_version=${RUNNER_VERSION}" + echo "compile_sdk=${COMPILE_SDK}" + } >> "$GITHUB_OUTPUT" - name: Login to GitHub Container Registry uses: docker/login-action@v4 @@ -82,14 +99,22 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/github-runner tags: | - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}${{ matrix.variant == 'ndk' && '-ndk' || '' }},enable=${{ matrix.variant != 'slim' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} - name: Build and push by digest id: build uses: docker/build-push-action@v7 with: context: . + target: ${{ matrix.target }} platforms: ${{ matrix.platform }} + # Layer cache in the GitHub Actions cache, one scope per platform + # (the ndk and slim variants keep their own scope and read the standard one). + cache-from: | + type=gha,scope=${{ matrix.arch }} + ${{ matrix.cache_scope != matrix.arch && format('type=gha,scope={0}', matrix.cache_scope) || '' }} + cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} # Disable SLSA provenance: it wraps the image in a per-registry # attestation index which produces unknown/unknown manifest entries # and an extra attestation manifest per arch. Nothing in this @@ -102,7 +127,10 @@ jobs: NODE_VERSION=${{ matrix.node_version }} RUNNER_IMAGE_TAG=${{ steps.meta.outputs.version }} RUNNER_IMAGE_REVISION=${{ github.sha }} - outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/github-runner,push-by-digest=true,name-canonical=true,push=true + # zstd layers pull and decompress faster on small hosts (Raspberry Pi) than gzip. + # force-compression recompresses cached/base layers too; oci-mediatypes is + # required for zstd. `imagetools create` below merges the per-arch OCI manifests. + outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/github-runner,push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3,force-compression=true,oci-mediatypes=true - name: Export digest run: | @@ -114,15 +142,16 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v7 with: - name: digests-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-${{ matrix.arch }} + name: digests-${{ matrix.variant }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-${{ matrix.arch }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 merge: - name: Merge JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} + name: Merge JDK ${{ matrix.java_version }} Node ${{ matrix.node_version }} (${{ matrix.variant }}) needs: [matrix, build] runs-on: ubuntu-latest + timeout-minutes: 15 strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.merge_matrix) }} @@ -136,19 +165,25 @@ jobs: RUNNER_TAG=$(grep -m1 '^ARG VERSION=' Dockerfile | cut -d= -f2) RUNNER_VERSION=$(echo "$RUNNER_TAG" | cut -d- -f1) COMPILE_SDK=$(grep -m1 '^ARG COMPILE_SDK=' Dockerfile | cut -d= -f2) + BUILD_TOOLS=$(grep -m1 '^ARG BUILD_TOOLS=' Dockerfile | cut -d= -f2) + NDK_VERSION=$(grep -m1 '^ARG NDK_VERSION=' Dockerfile | cut -d= -f2) - if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ]; then + if [ -z "$RUNNER_TAG" ] || [ -z "$RUNNER_VERSION" ] || [ -z "$COMPILE_SDK" ] || [ -z "$BUILD_TOOLS" ] || [ -z "$NDK_VERSION" ]; then echo "::error::Failed to extract versions from Dockerfile" exit 1 fi - echo "runner_version=${RUNNER_VERSION}" >> $GITHUB_OUTPUT - echo "compile_sdk=${COMPILE_SDK}" >> $GITHUB_OUTPUT + { + echo "runner_version=${RUNNER_VERSION}" + echo "compile_sdk=${COMPILE_SDK}" + echo "build_tools=${BUILD_TOOLS}" + echo "ndk_version=${NDK_VERSION}" + } >> "$GITHUB_OUTPUT" - name: Download digests uses: actions/download-artifact@v8 with: - pattern: digests-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-* + pattern: digests-${{ matrix.variant }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-* merge-multiple: true path: ${{ runner.temp }}/digests @@ -168,11 +203,22 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/github-runner tags: | - type=raw,value=jdk${{ matrix.java_version }},enable=${{ matrix.is_default_node }} - type=raw,value=node${{ matrix.node_version }},enable=${{ matrix.is_default_jdk }} - type=raw,value=jdk${{ matrix.java_version }}-node${{ matrix.node_version }} - type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }} - type=raw,value=latest,enable=${{ matrix.is_default_jdk && matrix.is_default_node }} + type=raw,value=jdk${{ matrix.java_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_node }} + type=raw,value=node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' && matrix.is_default_jdk }} + type=raw,value=jdk${{ matrix.java_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} + # Every Gradle-pinnable SDK component in the tag (compileSdk, build-tools), so + # consumers can let Renovate derive them from the image pin. + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'std' }} + type=raw,value=latest,enable=${{ matrix.variant == 'std' && matrix.is_default_jdk && matrix.is_default_node }} + type=raw,value=ndk,enable=${{ matrix.variant == 'ndk' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }}-ndk,enable=${{ matrix.variant == 'ndk' }} + # Only the ndk variant ships an NDK, so only it spells the NDK version out + # (upstream's -sdk-ndk- and -sdk-bt-ndk- tags; Gradle ndkVersion via Renovate). + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-bt${{ steps.versions.outputs.build_tools }}-ndk${{ steps.versions.outputs.ndk_version }}-node${{ matrix.node_version }},enable=${{ matrix.variant == 'ndk' }} + type=raw,value=slim,enable=${{ matrix.variant == 'slim' }} + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-node${{ matrix.node_version }}-slim,enable=${{ matrix.variant == 'slim' }} - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests @@ -183,6 +229,8 @@ jobs: TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") # Source: one per arch, by digest, from the GHCR push in the build job. SOURCES=$(printf "${IMAGE}@sha256:%s " *) + # TAGS and SOURCES are deliberately word-split into arguments. + # shellcheck disable=SC2086 docker buildx imagetools create $TAGS $SOURCES for tag in $(jq -cr '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON"); do diff --git a/Dockerfile b/Dockerfile index d90f1a4..aac655a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,93 +1,151 @@ ARG VERSION=2.338.0-ubuntu-noble ARG JAVA_VERSION=21 ARG NODE_VERSION=24 -ARG COMPILE_SDK=36.1 +ARG COMPILE_SDK=37.2 ARG BUILD_TOOLS=37.0.0 ARG NDK_VERSION=30.0.16248370 ARG SDK_TOOLS=8512546_latest ARG ANDROID_ROOT=/usr/local/lib/android +# JDK that sdkmanager runs on in the sdk stages. Fixed on purpose and independent +# of JAVA_VERSION: the SDK layer must come out identical for every JDK/Node tag, +# so it is shared in the registry and on hosts instead of stored once per tag. +ARG SDK_JAVA_VERSION=21 ################################################################################ -# base image from https://github.com/myoung34/docker-github-actions-runner -# - install temurin java into it -################################################################################ -FROM myoung34/github-runner:$VERSION AS java -ARG JAVA_VERSION -ARG NODE_VERSION -# TARGETARCH is set automatically by Buildx (e.g. amd64, arm64) and matches the -# arch suffix Temurin uses on its apt-installed JDK directory. -ARG TARGETARCH - -# https://adoptium.net/installation/linux -# add the gpp key and apt repo -RUN mkdir -p /etc/apt/keyrings && \ - wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc && \ - echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list - -# install the temurin jdk -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends temurin-$JAVA_VERSION-jdk -ENV JAVA_HOME=/usr/lib/jvm/temurin-$JAVA_VERSION-jdk-$TARGETARCH - -# install ip tools -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iproute2 - -# install cmake build-essential -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cmake build-essential swig - -# install nodejs from NodeSource. Replaces the base image's Ubuntu-default node -# (currently node 18 on noble) with a current LTS that user workflows can rely -# on without an extra setup-node step. -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends nodejs \ - && rm -rf /var/lib/apt/lists/* - -################################################################################ -# - install android sdk +# sdk: the Android SDK, built once for every tag +# - runs on a fixed JDK (SDK_JAVA_VERSION), never on JAVA_VERSION/NODE_VERSION +# - only the build args below reach this stage, so a JDK or Node bump does not +# invalidate it ################################################################################ -FROM java AS android +FROM eclipse-temurin:${SDK_JAVA_VERSION}-jdk-noble AS sdk ARG COMPILE_SDK ARG BUILD_TOOLS -ARG NDK_VERSION ARG SDK_TOOLS ARG ANDROID_ROOT -WORKDIR /tmp - -# download android tools and use it to install the SDK -RUN mkdir -p ${ANDROID_ROOT}/sdk/cmdline-tools/latest -RUN wget -O android-sdk.zip https://dl.google.com/android/repository/commandlinetools-linux-$SDK_TOOLS.zip && unzip android-sdk.zip && cp -r ./cmdline-tools/* ${ANDROID_ROOT}/sdk/cmdline-tools/latest -RUN ${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager --licenses >/dev/null +RUN apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends unzip wget \ + && rm -rf /var/lib/apt/lists/* -# Install latest Android SDK components only. Older versions are not needed since -# build-tools and NDK are backward compatible, and you only need the platform -# matching your project's compileSdk. Projects needing older platforms can install -# them at build time via sdkmanager. -# # sdkmanager has no download retry: a truncated download from dl.google.com # surfaces as "Archive is not a ZIP archive" and fails the whole build. It is # idempotent (already-installed packages are skipped), so retry the whole # command a few times, clearing its download scratch dir between attempts. -RUN attempt=1; \ - until echo "y" | ${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager --sdk_root=$ANDROID_ROOT/sdk/ \ +# Usage: sdk-install ... +RUN cat > /usr/local/bin/sdk-install <<'EOF' && chmod +x /usr/local/bin/sdk-install +#!/bin/sh +set -u +sdk_root=$1 +shift +attempt=1 +until echo "y" | "${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager" --sdk_root="$sdk_root" "$@"; do + if [ "$attempt" -ge 3 ]; then + echo "sdkmanager failed after $attempt attempts" >&2 + exit 1 + fi + echo "sdkmanager attempt $attempt failed, retrying in 15s..." >&2 + rm -rf "$sdk_root/.temp" + attempt=$((attempt + 1)) + sleep 15 +done +EOF + +# download android tools and use it to install the SDK +WORKDIR /tmp +RUN mkdir -p ${ANDROID_ROOT}/sdk/cmdline-tools/latest \ + && wget -q -O android-sdk.zip https://dl.google.com/android/repository/commandlinetools-linux-$SDK_TOOLS.zip \ + && unzip -q android-sdk.zip \ + && cp -r ./cmdline-tools/* ${ANDROID_ROOT}/sdk/cmdline-tools/latest \ + && rm -rf android-sdk.zip cmdline-tools +RUN ${ANDROID_ROOT}/sdk/cmdline-tools/latest/bin/sdkmanager --licenses >/dev/null + +# Install latest Android SDK components only. Older versions are not needed since +# build-tools are backward compatible, and you only need the platform matching +# your project's compileSdk. Projects needing older platforms can install them at +# build time via sdkmanager. The NDK and cmake live in the opt-in sdk-ndk stage. +RUN sdk-install ${ANDROID_ROOT}/sdk \ "platform-tools" \ "platforms;android-${COMPILE_SDK}" \ "build-tools;${BUILD_TOOLS}" \ + && rm -rf ${ANDROID_ROOT}/sdk/.temp + +################################################################################ +# sdk-ndk: NDK and the SDK's own cmake, for the opt-in runner-ndk target +# - installed into a separate sdk root, so the final image gets only the delta +# (ndk/ and cmake/) instead of a second full copy of the SDK +################################################################################ +FROM sdk AS sdk-ndk +ARG NDK_VERSION +ARG ANDROID_ROOT + +RUN mkdir -p /ndk-sdk && cp -r ${ANDROID_ROOT}/sdk/licenses /ndk-sdk/licenses +RUN sdk-install /ndk-sdk \ "ndk;${NDK_VERSION}" \ - "cmake;3.22.1"; do \ - if [ "$attempt" -ge 3 ]; then echo "sdkmanager failed after $attempt attempts" >&2; exit 1; fi; \ - echo "sdkmanager attempt $attempt failed, retrying in 15s..." >&2; \ - rm -rf ${ANDROID_ROOT}/sdk/.temp; \ - attempt=$((attempt + 1)); \ - sleep 15; \ - done + "cmake;3.22.1" \ + && rm -rf /ndk-sdk/.temp -WORKDIR /actions-runner -ENV PATH="${PATH}:/usr/local/lib/android/sdk/platform-tools/" +################################################################################ +# core: everything every runner target shares (and, as is, runner-slim) +# base image from https://github.com/myoung34/docker-github-actions-runner +# - the tag-independent apt packages first, then the JDK, then Node +################################################################################ +FROM myoung34/github-runner:$VERSION AS core +# Packages that do not depend on JAVA_VERSION/NODE_VERSION, so this layer is +# shared by every tag too. +# - iproute2: ip tools +# - cmake, build-essential, swig: native builds +# - xxd: not in the base image +# - pigz: parallel gzip, used by tar/docker when available +RUN apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + iproute2 cmake build-essential swig xxd pigz \ + && rm -rf /var/lib/apt/lists/* -# Set env variable for SDK Root (https://developer.android.com/studio/command-line/variables) -# ANDROID_HOME is deprecated, but older versions of Gradle rely on it -ENV ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk -ENV ANDROID_HOME=$ANDROID_ROOT/sdk +# rustup-init only, no toolchain: ~/.cargo and ~/.rustup are links into the +# runner's slot, so anything installed there at build time would be hidden. +# The preflight runs it once per slot (`--default-toolchain none`, offline), so +# `rustup` is on PATH and the toolchains a job installs (dtolnay/rust-toolchain, +# rust-toolchain.toml) land in the warm slot cache. The checksum comes from the +# same versioned archive. +ARG RUSTUP_VERSION=1.29.1 +# TARGETARCH is set automatically by Buildx (e.g. amd64, arm64). It also matches +# the arch suffix Temurin uses on its apt-installed JDK directory (JAVA_HOME). +ARG TARGETARCH +RUN case "$TARGETARCH" in \ + amd64) triple=x86_64-unknown-linux-gnu ;; \ + arm64) triple=aarch64-unknown-linux-gnu ;; \ + *) echo "unsupported arch $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${triple}/rustup-init" \ + && curl -fsSL --retry 3 -o /usr/local/bin/rustup-init "$url" \ + && echo "$(curl -fsSL --retry 3 "$url.sha256" | cut -d' ' -f1) /usr/local/bin/rustup-init" | sha256sum -c - \ + && chmod 755 /usr/local/bin/rustup-init +ENV PATH="/root/.cargo/bin:${PATH}" + +# JAVA_VERSION and NODE_VERSION are declared only now, below the layers every tag +# shares: a build arg is part of the cache key of each RUN that follows its ARG, +# so declaring them at the top would rebuild (and re-hash) those layers per tag. +ARG JAVA_VERSION +ARG NODE_VERSION + +# https://adoptium.net/installation/linux +# add the gpg key and apt repo, then install the temurin jdk +RUN mkdir -p /etc/apt/keyrings \ + && wget -qO /etc/apt/keyrings/adoptium.asc https://packages.adoptium.net/artifactory/api/gpg/key/public \ + && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ + && apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends temurin-$JAVA_VERSION-jdk \ + && rm -rf /var/lib/apt/lists/* +ENV JAVA_HOME=/usr/lib/jvm/temurin-$JAVA_VERSION-jdk-$TARGETARCH + +# install nodejs from NodeSource. Replaces the base image's Ubuntu-default node +# (currently node 18 on noble) with a current LTS that user workflows can rely +# on without an extra setup-node step. +RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /actions-runner LABEL maintainer="asen.lekov@dodi.li" # Jobs run as root here, and GNU tar run as root keeps the owner ids stored in an @@ -102,9 +160,9 @@ LABEL maintainer="asen.lekov@dodi.li" ENV TAR_OPTIONS=--no-same-owner # Tool caches that outlive a job. Jobs run as root (HOME=/root), so these are -# the tools' own defaults made explicit: docker-compose.yml mounts one named -# volume on each of them. Mount the specific directories, never ~/.cache as a -# whole, or it would shadow the pnpm volume at ~/.cache/pnpm. +# the tools' own defaults made explicit: the preflight links each of them into +# the runner's slot (see preflight-entrypoint.sh). Link the specific +# directories, never ~/.cache as a whole, or it would shadow ~/.cache/pnpm. # - mise: tool installs (data) and downloads (cache). jdx/mise-action and # `mise install` fill these. # - hk: HK_CACHE_DIR is its config/tool cache. HK_PKL_CACHE_DIR defaults to @@ -115,37 +173,87 @@ ENV MISE_DATA_DIR=/root/.local/share/mise \ HK_CACHE_DIR=/root/.cache/hk \ HK_PKL_CACHE_DIR=/root/.cache/hk/pklr -# Job hooks: JOB_STARTED is the poison sweep (covers OOM / killed runner); -# JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches. -COPY pre-job.sh /usr/local/bin/pre-job.sh -COPY cleanup.sh /usr/local/bin/cleanup.sh -# pruner.sh is not a hook: the `pruner` compose service runs it nightly -# (LRU cache caps + docker prune), keeping those scans off the job path. -COPY pruner.sh /usr/local/bin/pruner.sh -RUN chmod +x /usr/local/bin/pre-job.sh /usr/local/bin/cleanup.sh /usr/local/bin/pruner.sh -ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh -ENV ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh - -# Preflight entrypoint: validates the persisted runner registration against -# GitHub before handing off to the upstream entrypoint. Upstream reuses -# CONFIGURED_ACTIONS_RUNNER_FILES_DIR credentials unconditionally, which -# crash-loops the container when the runner was removed server-side; the -# preflight wipes provably-dead registrations so a fresh ACCESS_TOKEN -# registration happens instead. See preflight-entrypoint.sh for details. -COPY preflight-entrypoint.sh /usr/local/bin/preflight-entrypoint.sh -RUN chmod +x /usr/local/bin/preflight-entrypoint.sh -# Image identity. deploy.yml passes the image tag and the commit it was built -# from; jobs read them (e.g. the runner report) to tell which image they ran -# on. They default to "dev" for local builds. Declared last: the values change -# on every build, so anything after them would never hit the layer cache. -ARG RUNNER_IMAGE_TAG=dev -ARG RUNNER_IMAGE_REVISION=dev -ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ - RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION +# Job hooks: JOB_STARTED marks the slot busy (and sweeps up after a killed +# job); JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches. +# The scripts themselves come from the `scripts` stage, copied last into each +# target (see there). +ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh \ + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh +# preflight-entrypoint.sh claims the runner's slot, links its caches, then +# validates a persisted registration before handing off to the upstream +# entrypoint (see the script for details). ENTRYPOINT ["/usr/local/bin/preflight-entrypoint.sh"] # Declaring ENTRYPOINT resets the CMD inherited from the base image, so the # base image's CMD is restated here. See # https://github.com/myoung34/docker-github-actions-runner#environment-variables # for how to use the image CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] + +################################################################################ +# scripts: this repo's runtime scripts, the files edited most often. Every +# target copies them as its LAST layer before the identity, so editing one +# never invalidates (or re-pushes) the SDK and NDK layers below. +# - pruner.sh is not a hook: the `pruner` compose service runs it (cache +# budgets + docker prune), keeping those scans off the job path +# - --chmod sets the mode in the COPY layer itself +################################################################################ +FROM scratch AS scripts +COPY --chmod=755 preflight-entrypoint.sh pre-job.sh cleanup.sh pruner.sh /usr/local/bin/ + +################################################################################ +# runner-slim (`--target runner-slim`): core without the Android SDK +# - for arm64 hosts (Android's build tools are x86_64-only), JS/JVM-only pools +# and small disks +################################################################################ +FROM core AS runner-slim +COPY --from=scripts / / +# Image identity. deploy.yml passes the image tag and the commit it was built +# from; jobs read them (e.g. the runner report) to tell which image they ran +# on. They default to "dev" for local builds. Declared last in every target +# (the same three lines close runner-ndk and runner): the values change on +# every build, so anything after them would never hit the layer cache. +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION + +################################################################################ +# android: core + the Android SDK, the base of runner and runner-ndk +# - the SDK sits on top of the JDK/Node layers: the weekly SDK bump then only +# replaces this layer, and hosts keep the JDK/Node layers they already have. +# Its blob is the same for every JDK/Node tag (the sdk stage never sees +# JAVA_VERSION/NODE_VERSION), so the registry stores and CI builds it once. +################################################################################ +FROM core AS android +ARG ANDROID_ROOT +COPY --from=sdk ${ANDROID_ROOT} ${ANDROID_ROOT} +# Set env variable for SDK Root (https://developer.android.com/studio/command-line/variables) +# ANDROID_HOME is deprecated, but older versions of Gradle rely on it +ENV ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk \ + ANDROID_HOME=$ANDROID_ROOT/sdk \ + PATH="${PATH}:${ANDROID_ROOT}/sdk/platform-tools/" + +################################################################################ +# runner-ndk (opt-in, `--target runner-ndk`): android + NDK + the SDK's cmake +# - the NDK and cmake;3.22.1 are several GB that most consumers never use +################################################################################ +FROM android AS runner-ndk +ARG ANDROID_ROOT +COPY --from=sdk-ndk /ndk-sdk/ndk ${ANDROID_ROOT}/sdk/ndk +COPY --from=sdk-ndk /ndk-sdk/cmake ${ANDROID_ROOT}/sdk/cmake +COPY --from=scripts / / +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION + +################################################################################ +# runner (default, MUST stay the last stage so a plain `docker build` makes it) +################################################################################ +FROM android AS runner +COPY --from=scripts / / +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION diff --git a/README.md b/README.md index 24a7093..080744b 100644 --- a/README.md +++ b/README.md @@ -1,190 +1,392 @@ -# docker-github-actions-runner-android +# GitHub Actions runner for Android, JVM and JavaScript -A self-hosted GitHub Actions runner image, plus a Docker Compose setup for -running it at scale. The image builds on -[myoung34/github-runner](https://github.com/myoung34/docker-github-actions-runner) -and adds: +A self-hosted GitHub Actions runner image, plus a Docker Compose setup that +runs a fleet of them on anything from a Raspberry Pi 5 to a 64 GB build box. -- a Temurin JDK, the Android SDK and Node.js LTS, so jobs can skip - `setup-java`, `setup-android` and `setup-node`; -- multi-arch images (`linux/amd64`, `linux/arm64`). +```bash +docker pull ghcr.io/dodi-smart/github-runner:latest +``` -The Compose setup adds: +**The image** builds on +[myoung34/github-runner](https://github.com/myoung34/docker-github-actions-runner) +and adds: -- ephemeral runners with persistent cache volumes; -- a `pruner` service that does the maintenance; -- a local action archive cache that avoids GitHub's download rate limits. +- the Temurin JDK, the Android SDK (platform, build-tools, platform-tools) and + Node.js LTS, so jobs can skip `setup-java`, `setup-android` and `setup-node`; +- the usual CLI tools jobs reach for: `git`, `git-lfs`, `jq`, `yq`, `gh`, + `zip`/`unzip`, `zstd`, `xxd`, `rsync` and `docker`; +- `rustup` on `PATH` without a toolchain: Rust toolchains install on first + use (`dtolnay/rust-toolchain`, `rust-toolchain.toml`) into the runner's + cache and are reused from then on, at no cost to the image size; +- multi-arch images (`linux/amd64`, `linux/arm64`), in three variants: + `slim`, the default with the Android SDK, and `ndk`. + +**The Compose setup** adds: + +- **ephemeral runners**: one job per container, then a clean restart; +- **warm caches that survive**: per-runner caches for bun, npm, pnpm, Gradle, + Cargo/rustup, Kotlin/Native, mise, hk, the Dart pub cache and the hosted + tool cache; +- **one setting to size a host**, `RUNNERS=N`, plus a script that suggests it; +- **`docker run -v` and container actions that work**, because work dirs sit + at the same path on the host and in the runner; +- **a pruner** that keeps every cache within a disk budget sized to the + machine, without touching a cache while a job is using it; +- **a local action archive cache** that avoids GitHub's download rate limits. + +## Contents + +- [Quick start](#quick-start) +- [Sizing a host](#sizing-a-host) +- [Image variants and tags](#image-variants-and-tags) +- [How it works](#how-it-works) +- [Using it from workflows](#using-it-from-workflows) +- [Caches](#caches) +- [Maintenance](#maintenance) +- [Action archive cache](#action-archive-cache) +- [Configuration reference](#configuration-reference) +- [Troubleshooting](#troubleshooting) +- [Upgrading from named volumes](#upgrading-from-named-volumes) +- [Development](#development) ## Quick start -1. Create a GitHub App for runner registration. See the - [base image docs](https://github.com/myoung34/docker-github-actions-runner#environment-variables). -2. Copy [`.env.example`](.env.example) to `.env` and set `APP_ID`, - `APP_PRIVATE_KEY` and `APP_LOGIN`. -3. Start the runners: - ```bash - docker compose up -d - ``` - -[`docker-compose.yml`](docker-compose.yml) starts three runners and a `pruner`. -Each runner is **ephemeral**: it takes one job, then restarts and registers -again with a clean container. Only the named volumes survive between jobs. - -To update a host, run `docker compose pull && docker compose up -d`. - -## Fit it to your setup - -**One host or many.** Run the same Compose file on every host and change only -`.env`. To run more or fewer runners per host, add or remove `runner-N` -services, along with their volumes. - -**Different kinds of machines.** Give each host the capability labels your -workflows select on, through `LABELS` in its `.env`. For example, `build` for -general jobs and `heavy` for memory-hungry ones. Workflows then ask for -`runs-on: [self-hosted, Linux, build]`. Label by what a machine can do, not by -its name. A host can carry several labels, so a large machine labelled -`build,heavy` also picks up general jobs when the others are busy. - -**One runner in another pool.** `RUNNER__LABELS` (for `runner`, `runner-2`, -`runner-3`) replaces `LABELS` for that runner alone. A host whose runners are -`light` can let one of them also build, with -`RUNNER_3_LABELS=docker,x86,light,large`, without a second host file. - -**Coordination runners.** `COMPOSE_PROFILES=control` adds two runners that -carry only `CONTROL_LABELS` (default `control`). Point the short jobs a merge -waits on at them, such as a picker or a required summary job, so those never -queue behind long builds. They hold no `docker.sock` and no tool cache, so keep -jobs that check out and build code off them. - -**Small hosts.** Boards such as a Raspberry Pi have little disk. Layer -[`hosts/no-caches.compose.yml`](hosts/no-caches.compose.yml) by setting -`COMPOSE_FILE` in `.env`. Those runners skip the per-runner cache volumes. - -**Repository or enterprise runners.** The Compose file registers runners for -an organization (`RUNNER_SCOPE: org`). For other scopes, change the runner -environment to what the -[base image](https://github.com/myoung34/docker-github-actions-runner#environment-variables) -supports. - -**Your own image.** Set `RUNNER_IMAGE` to use a fork or a pinned tag. See -[Image tags](#image-tags). +You need a Linux host with Docker Engine and the Compose plugin, and admin +rights on a GitHub organization. -## Caches +**1. Create a GitHub App** that registers the runners. In your organization, +go to *Settings → Developer settings → GitHub Apps → New GitHub App*: -Each runner gets its own volume for these caches: +- **Organization permissions → Self-hosted runners:** Read and write. +- **Repository permissions → Contents:** Read-only. This is only needed if the + [action archive cache](#action-archive-cache) should read private + repositories. +- Turn off the webhook. Install the App on the organization, note its **App + ID**, and generate a **private key**. -- bun, npm and pnpm -- Gradle -- Cargo and rustup -- Kotlin/Native -- mise and hk +**2. Get the Compose files** onto the host: -The image sets `MISE_DATA_DIR`, `MISE_CACHE_DIR` and `HK_CACHE_DIR` so those -tools write there. +```bash +git clone https://github.com/dodi-smart/docker-github-actions-runner-android.git runners +cd runners +cp .env.example .env +``` -- **Never share a cache volume between runners.** Concurrent jobs would - corrupt it. -- **Keep the default cache locations in workflows.** If a job moves - `GRADLE_USER_HOME` or `BUN_INSTALL_CACHE_DIR` into `$RUNNER_TEMP`, the - volumes go unused. -- **Turn off the GitHub cache where a volume replaces it.** For example, use - `cache: false` on `jdx/mise-action`. -- **Don't put `/tmp` on tmpfs.** The job workspace is `/tmp/runner/work`. +**3. Size the host.** The script only reads; it changes nothing. Choose +`light` for checks, lint and JS builds, or `large` for Gradle and Android +builds: -## Action archive cache +```bash +hosts/sizing.sh light +``` -Ephemeral runners download every `uses:` action on every job. Several runners -behind one IP address soon hit `429 Too Many Requests` in "Set up job". +Paste its suggestions (`RUNNERS`, `RUNNER_CPUS`, `RUNNER_MEMORY`, and +`RUNNER_IMAGE` on arm64) into `.env`. -The runner can read action archives from a local directory before it downloads -anything. Set `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE` to that directory. Here it -is `/opt/actionarchivecache`, mounted read-only into every runner. The runner -never writes this directory itself, so the `pruner` fills it every hour. It -uses the GitHub App's token, so the downloads count against the App's rate -limit rather than your IP address. It fetches: +**4. Fill in `.env`.** At minimum set `APP_ID`, `APP_PRIVATE_KEY` (the PEM on +one line, with `\n` for newlines) and `APP_LOGIN` (the organization). Then set +the labels your workflows will select on, for example `LABELS=docker,light`. -- every `uses:` found in the repositories listed in `PREFETCH_SOURCES`, such as - the repo that holds your reusable workflows; -- the actions listed in `PREFETCH_EXTRA`. +**5. Start the fleet:** + +```bash +docker compose up -d +docker compose logs -f runner # each runner logs the slot it claimed +``` -With neither variable set, the cache stays empty and jobs download as usual. A -cache miss always falls back to a normal download. +The runners show up under the organization's *Settings → Actions → Runners*. +Point a workflow at them with `runs-on: [self-hosted, Linux, light]`. -To fill the cache straight away: +To update a host later: ```bash -docker compose run --rm pruner prefetch +git pull && docker compose pull && docker compose up -d ``` -To confirm it works, re-run a job with debug logging. "Set up job" then shows -`Copied action archive '/opt/actionarchivecache/...'`. +## Sizing a host + +Every host runs the same Compose file. Only `.env` differs. `hosts/sizing.sh` +reads the host's CPUs, memory and disk: + +- it fits as many runners as both CPU and memory allow for the chosen profile; +- `light` gives each runner 1 CPU and 3 GiB, `large` gives 2 CPUs and 6 GiB; +- it keeps 2 GiB of memory for the host itself. + +Typical results: + +| Host | Profile | Suggested `.env` | +|---|---|---| +| Raspberry Pi 5, 8 GB (arm64) | light | `RUNNERS=2`, `RUNNER_IMAGE=…:slim`. The cache budget shrinks with the disk on its own. | +| Hetzner CPX31, 4 vCPU / 8 GB | light | `RUNNERS=2`, `RUNNER_CPUS=1`, `RUNNER_MEMORY=3g` | +| Hetzner CX43, 8 vCPU / 16 GB | light or large | `RUNNERS=4` (light) or `RUNNERS=2` (large) | +| 64 GB mini PC / build box | large | as many as CPU allows, e.g. `RUNNERS=6`, `RUNNER_CPUS=2`, `RUNNER_MEMORY=6g` | + +What the knobs do: + +- **`RUNNER_CPUS` / `RUNNER_MEMORY`** cap each runner, so one heavy job cannot + starve its neighbours. Under a CPU cap the runner also caps Gradle + (`org.gradle.workers.max`), Cargo, `make` and CMake parallelism to it. A + container otherwise sees every host CPU and spawns a worker for each. Leave + both unset, or `0`, for no limit. +- **Labels** describe capability, not machine names. A host can carry several, + so a big machine labelled `light,large` also picks up light jobs. + `RUNNER__LABELS` gives runner slot *n* its own set, for example one + `large` runner on an otherwise `light` host. +- **Coordination runners:** `CONTROL_RUNNERS=2` adds runners that carry only + `CONTROL_LABELS` (default `control`). They have no Docker socket and no + caches. Point the short jobs a merge waits on (pickers, required summary + jobs) at them, so those never queue behind builds. +- **Tiny disks:** the pruner's budget already scales with the disk. If even + that is too much, `RUNNER_CACHES=off` runs without caches. + +## Image variants and tags + +Images are published to `ghcr.io/dodi-smart/github-runner` for `linux/amd64` +and `linux/arm64`. + +| Variant | Contents | Use it for | +|---|---|---| +| `slim` | JDK, Node, CLI tools; **no Android SDK** | arm64 hosts (Android's build tools are x86_64-only), JS/JVM-only pools, small disks | +| default | `slim` + Android SDK (platform, build-tools, platform-tools) | Android and general CI | +| `ndk` | default + NDK and CMake | projects with native (C/C++) code | + +The Android SDK sits in one layer that is identical across every JDK and Node +tag, so a host running several tags downloads it once. + +| Tag | Pins | +|---|---| +| `latest` | nothing: newest JDK + Node LTS, default variant | +| `jdk21`, `node22` | that JDK or Node major | +| `jdk21-node22` | both majors | +| `slim`, `ndk` | the variant, with the default JDK + Node | +| `2.338.0-jdk21-sdk37.2-node24` | everything: runner, JDK, SDK and Node. Never overwritten | +| `2.338.0-jdk21-sdk37.2-bt37.0.0-node24` | the same image with build-tools spelled out | + +The long tags let a consumer have Renovate derive its Gradle `compileSdk` and +`buildToolsVersion` from the image pin, so AGP doesn't download SDK components +on every job. + +JDKs: the LTS lines (17, 21, 25) plus the newest GA release. Node: the current +LTS lines. [`matrix.json`](matrix.json) is the source of truth. A weekly job +proposes new JDK, Node, SDK, build-tools and NDK versions. Each image carries +`RUNNER_IMAGE_TAG` and `RUNNER_IMAGE_REVISION`, so a job can tell what it ran +on. + +A project that needs another Android platform installs it in the job, e.g. +`sdkmanager "platforms;android-34"`. + +## How it works + +```mermaid +flowchart LR + subgraph host["Host: RUNNER_DATA_DIR (/var/lib/github-runner)"] + s1["slots/1
work/ · cache/"] + s2["slots/2
work/ · cache/"] + sn["slots/N …"] + end + r1["runner container"] -- "flock .lock" --> s1 + r2["runner container"] -- "flock .lock" --> s2 + p["pruner"] -. "budget, skips busy slots" .-> host + d[("host Docker daemon")] + r1 -- "docker.sock: docker run -v $PWD" --> d + d -- "same path" --> s1 +``` + +- **Slots.** Each runner container claims the lowest free slot under + `RUNNER_DATA_DIR/slots/` and holds a lock on it for its whole life. Its work + dir and every cache live in that slot, so no two running jobs ever share a + cache. Scaling to `RUNNERS=N` just means N slots. +- **Ephemeral.** A runner takes one job, deregisters and exits. + `restart: unless-stopped` brings it back to register afresh. `/tmp` and + other leftovers are wiped after each job; the slot's caches stay. +- **Same path on host and container.** `RUNNER_DATA_DIR` is mounted at the + same path inside the containers. When a job hands a path to the host's + Docker daemon (`docker run -v "$PWD:/src"`, a container action, a + `container:` job), the daemon finds the same files. +- **Cache links.** The tools' default locations (`~/.gradle`, `~/.bun`, + `/opt/hostedtoolcache`, …) are symlinks into the slot. Workflows don't need + any cache configuration. +- **Self-healing registration.** + [`preflight-entrypoint.sh`](preflight-entrypoint.sh) wipes a stored runner + registration only after GitHub confirms that runner is gone (`404`). A + deleted runner then registers again instead of crash-looping. + +## Using it from workflows + +```yaml +jobs: + build: + runs-on: [self-hosted, Linux, large] + steps: + - uses: actions/checkout@v4 + - run: ./gradlew assembleDebug # JDK, SDK and ~/.gradle are warm already +``` + +- **Keep the tools' default cache locations.** If a job moves + `GRADLE_USER_HOME`, `BUN_INSTALL_CACHE_DIR` or `PUB_CACHE` into + `$RUNNER_TEMP`, the warm caches go unused. +- **Skip the GitHub cache where a runner cache replaces it,** e.g. + `cache: false` on `jdx/mise-action` and `setup-node`. With `actions/cache` + and the bun cache, use an exact key (no `restore-keys`), so a bad entry + cannot come back. +- **Docker works.** Use `docker build`, `docker run -v`, service containers + and container actions as on GitHub-hosted runners. Ports a job's containers + publish open on the host, not on the runner's `localhost`. +- **Jobs run as root.** Tool archives unpack root-owned + (`TAR_OPTIONS=--no-same-owner`), so git does not refuse a Flutter SDK it + unpacked. + +## Caches + +Each slot keeps one directory per tool under `slots//cache/`: + +| Store | Linked from | Evicted as | +|---|---|---| +| bun | `~/.bun` | whole packages; never `bin/` | +| npm | `~/.npm` | files (content-addressed) | +| pnpm | `~/.local/share/pnpm`, `~/.cache/pnpm` | the whole store | +| gradle | `~/.gradle` | whole dependency versions, transforms, wrapper dists | +| cargo | `~/.cargo` | crates, unpacked sources, git checkouts; never `bin/` | +| rustup | `~/.rustup` | whole toolchains; never the default. rustup itself is set up in `~/.cargo` on first start | +| konan | `~/.konan` | whole dependencies and prebuilt compilers | +| mise | `~/.local/share/mise`, `~/.cache/mise` | whole tool versions; downloads as files | +| hk | `~/.cache/hk` | files | +| pub | `~/.pub-cache` | whole packages | +| toolcache | `/opt/hostedtoolcache` | whole tool versions (setup-* actions, Flutter) | + +Entries are evicted whole wherever a partial entry would break a build. A +package folder missing one file still looks complete to bun. ## Maintenance The `pruner` service ([`pruner.sh`](pruner.sh)) keeps slow work off the job path: -- **Nightly:** trims the Gradle, bun, mise and hk caches to `CACHE_CAP_BYTES` - each (default 20 GiB), least recently used first. The bun cache drops whole - package folders (`install/cache/@@@@1`, by the atime of their - `package.json`) and never `bin/`: bun reuses a package folder as complete - whenever it exists, so a single evicted file would go missing from every - later install. mise tool versions are dropped whole too; the Gradle, mise - download and hk caches drop single files. It then - runs `docker system prune --filter until=24h` and - `docker builder prune --filter until=72h`. Named volumes are never pruned. -- **Hourly:** fills the action archive cache, capped at 2 GiB. - -After each job, [`cleanup.sh`](cleanup.sh) does only a quick tidy-up: `/tmp`, +- **Cache budget.** Each host gets `CACHE_BUDGET_BYTES` (default 60 GiB) or + `CACHE_BUDGET_PCT` of the data disk (default 40 %), whichever is smaller, + split evenly across the slots. Within a slot, least recently used entries + go first, across all stores. +- **Never under a running job.** A slot whose runner is in a job is skipped. + A job that starts while the pruner works on its slot waits for it, up to + `PRUNE_WAIT_SECONDS`. +- **Adaptive.** It runs daily, and early when free disk drops below + `PRUNE_FREE_PCT`. Then `docker system prune` / `docker builder prune` clean + old images and build cache, more aggressively when disk is tight. +- **Hourly:** it fills the action archive cache. + +After each job, [`cleanup.sh`](cleanup.sh) does a quick tidy-up only: `/tmp`, Gradle daemons and locks, and runner logs. -To run the nightly pass once by hand: +To run a maintenance pass by hand: ```bash docker compose run --rm -e PRUNE_RUN_ONCE=true pruner ``` -## Image tags +## Action archive cache -Images are published to `ghcr.io/dodi-smart/github-runner`. +Ephemeral runners download every `uses:` action for every job. Several +runners behind one IP address soon hit `429 Too Many Requests` in "Set up +job". -| Tag | Pins | -|---|---| -| `latest` | Nothing | -| `jdk21` | JDK major | -| `node22` | Node major | -| `jdk21-node22` | JDK and Node majors | -| `2.338.0-jdk21-sdk36-node24` | Everything: runner, JDK, SDK and Node. Never overwritten | +How the cache avoids that: -[`matrix.json`](matrix.json) lists the available JDK and Node versions and the -defaults. Each image sets `RUNNER_IMAGE_TAG` and `RUNNER_IMAGE_REVISION`, so a -job can tell which image it ran on. +- The runner reads archives from `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE` + (`/opt/actionarchivecache`, read-only in every runner) before downloading. +- The pruner fills that directory hourly, as the GitHub App. Downloads count + against the App's limit, not your IP. +- It fetches every `uses:` found in `PREFETCH_SOURCES` (e.g. the repo that + holds your reusable workflows), plus `PREFETCH_EXTRA`. +- A cache miss falls back to a normal download. -To use another Android platform, install it in the job: -`sdkmanager "platforms;android-28"`. +To fill it straight away: -## Notes +```bash +docker compose run --rm pruner prefetch +``` -- `EPHEMERAL` and `DISABLE_AUTO_UPDATE` count as set for any non-empty value, - `"false"` included. The upstream entrypoint only checks whether they are - empty. -- [`preflight-entrypoint.sh`](preflight-entrypoint.sh) wipes a stored runner - registration only after GitHub confirms that runner is gone (`404`). A - removed runner then registers again instead of crash-looping. +To confirm it works, re-run a job with debug logging. "Set up job" then shows +`Copied action archive '/opt/actionarchivecache/...'`. -## Development +## Configuration reference + +Set these in `.env`. [`.env.example`](.env.example) has them all, with +comments. + +| Variable | Default | Purpose | +|---|---|---| +| `APP_ID`, `APP_PRIVATE_KEY`, `APP_LOGIN` | — | GitHub App and organization (required) | +| `RUNNERS` | `3` | runners on this host | +| `RUNNER_DATA_DIR` | `/var/lib/github-runner` | host directory for slots (work dirs + caches) | +| `LABELS` | `docker` | labels for every runner | +| `RUNNER__LABELS` | — | labels for slot *n* only | +| `RUNNER_CPUS`, `RUNNER_MEMORY` | none | per-runner limits (e.g. `2`, `6g`) | +| `RUNNER_CACHES` | `on` | `off` runs without caches | +| `CONTROL_RUNNERS`, `CONTROL_LABELS` | `0`, `control` | coordination runners | +| `RUNNER_IMAGE` | `ghcr.io/dodi-smart/github-runner` | image or tag (e.g. `:slim`) | +| `RUNNER_NAME_PREFIX`, `RUNNER_GROUP` | `runner`, `Default` | runner name and group | +| `CACHE_BUDGET_BYTES`, `CACHE_BUDGET_PCT` | 60 GiB, `40` | pruner cache budget per host | +| `PRUNE_FREE_PCT`, `PRUNE_INTERVAL_SECONDS` | `15`, `86400` | when the pruner runs | +| `PREFETCH_SOURCES`, `PREFETCH_EXTRA` | — | action archive cache sources | +| `DISABLE_AUTO_UPDATE` | empty | any non-empty value (even `false`) disables runner self-update | + +`EPHEMERAL` and `DISABLE_AUTO_UPDATE` count as set for any non-empty value, +because the upstream entrypoint only checks whether they are empty. + +## Troubleshooting + +- **A runner restarts in a loop.** Run `docker compose logs runner`. Check the + App ID, the key (one line, `\n` escapes) and that the App is installed on + `APP_LOGIN` with *Self-hosted runners: Read and write*. +- **Which slot is a runner on?** The first log line says + `[preflight] slot (...)`. Inside a job, `$RUNNER_SLOT` and + `$RUNNER_SLOT_DIR` say the same. +- **The disk fills up.** Lower `CACHE_BUDGET_BYTES`, then run a maintenance + pass by hand (see [Maintenance](#maintenance)). `du -sh + $RUNNER_DATA_DIR/slots/*/cache/*` shows what is big. +- **A job fails with a corrupt cache.** Stop that slot's runner, delete the + store (`rm -rf $RUNNER_DATA_DIR/slots//cache/`) and start it + again. It refills on the next job. +- **Don't put `RUNNER_DATA_DIR` on NFS.** Slot locks use `flock`, which needs + a local filesystem. + +## Upgrading from named volumes + +Earlier versions used three hard-coded runner services, each with its own +named volumes. To move to slots: + +```bash +git pull +docker compose down +# set RUNNERS (and CONTROL_RUNNERS=2 if you used COMPOSE_PROFILES=control) in .env +docker compose up -d +``` -[`build.yml`](.github/workflows/build.yml) builds every pull request. Its -`All builds passed` check is the one to require. +What changes: + +- Caches start cold once and warm up within a few jobs. +- The old volumes stay until you delete them with + `docker volume ls -q | grep -E '_runner-(work|bun|npm|pnpm|gradle|cargo|rustup|konan|mise|hk)' | xargs docker volume rm`. +- `COMPOSE_PROFILES=control` and `hosts/no-caches.compose.yml` are gone, + replaced by `CONTROL_RUNNERS` and `RUNNER_CACHES=off`. + +## Development -Renovate keeps the base image and the actions up to date; `forkProcessing` is -on because this repo is a fork. A weekly workflow opens pull requests for new -JDK, Node LTS, Android SDK, build-tools and NDK versions. +- [`build.yml`](.github/workflows/build.yml) builds and smoke-tests every pull + request and runs the script tests. Its `All builds passed` check is the one + to require. +- [`deploy.yml`](.github/workflows/deploy.yml) publishes the images from + `main`. +- Renovate keeps the base image and the actions current (`forkProcessing` is + on: this repo is a fork of + [compscidr/docker-github-actions-runner-android](https://github.com/compscidr/docker-github-actions-runner-android)). -To build locally: +Build and test locally: ```bash -docker build --build-arg JAVA_VERSION=17 --build-arg NODE_VERSION=22 . +docker build --target runner --build-arg JAVA_VERSION=21 --build-arg NODE_VERSION=24 . +docker run --rm -v "$PWD:/w" -w /w ubuntu:24.04 bash -c 'for t in tests/*.sh; do bash "$t"; done' +shellcheck -x *.sh hosts/*.sh tests/*.sh ``` The build args and their defaults are at the top of the diff --git a/cleanup.sh b/cleanup.sh index 29dfb19..d77b8e1 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -2,88 +2,48 @@ # Post-job cleanup script for GitHub Actions runner. # Invoked automatically via ACTIONS_RUNNER_HOOK_JOB_COMPLETED after each job. # -# Persist package, Gradle/Cargo and mise/hk caches on the named volumes. This -# hook only does the CHEAP tidy-up that has to happen after every job. The -# expensive size-cap scans (du over multi-GB trees, LRU eviction) used to run -# here and cost 21-39 s per job (issue #16); they now live in pruner.sh, run -# nightly by the `pruner` compose service. JOB_STARTED (pre-job.sh) is the -# poison sweep for a killed runner process. +# Only the CHEAP tidy-up that has to happen after every job. The size caps +# live in pruner.sh, run by the `pruner` compose service (issue #16: they used +# to cost 21-39 s per job here). JOB_STARTED (pre-job.sh) sweeps up after a +# job whose runner was killed before this hook ran. # # Intentionally not using set -e: individual cleanup failures should not # prevent other cleanups from running. # -# KEEP: $HOME/.bun, $HOME/.npm, $HOME/.cache/pnpm, $HOME/.cache/yarn, -# $HOME/.cache/mise, $HOME/.local/share/mise, $HOME/.cache/hk, -# $HOME/.konan, $HOME/.rustup, -# Gradle modules-2 / build-cache-* / transforms-*, Cargo registry. -# DELETE: /tmp contents except $RUNNER_WORKDIR (and $RUNNER_TEMP if under -# /tmp), Gradle daemon/journals/locks, runner _diag logs. +# KEEP: the slot caches (see preflight-entrypoint.sh) and the work dir, which +# lives in the slot (RUNNER_DATA_DIR/slots//work), not under /tmp. +# DELETE: /tmp contents, Gradle daemon/journals/locks, runner _diag logs. -echo "[cleanup] Running post-job cleanup..." -BEFORE=$(df -h / | awk 'NR==2 {print $4}') - -# True when $1 is $2 or lives inside $2. -_inside() { - [ -n "$2" ] || return 1 - case "$1" in "$2"|"$2"/*) return 0 ;; esac - return 1 -} - -# True when $1 is a strict ancestor of $2. -_ancestor_of() { - [ -n "$2" ] || return 1 - case "$2" in "$1"/*) return 0 ;; esac - return 1 -} - -# RUNNER_WORKDIR is /tmp/runner/work in docker-compose.yml. A blanket -# `find /tmp -delete` wipes the work volume. Spare the workdir (and -# RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp. -# -# Walk top-down and never descend into a spared tree: the work volume holds -# checkouts plus node_modules, and `find -depth` (which disables -prune) used -# to walk every path in it on every job. Only strict ancestors of a spared -# path are entered, so the cost is the handful of entries outside the workdir. -clean_dir() { - local dir="$1" workdir="$2" temp="$3" path - while IFS= read -r -d '' path; do - if _inside "$path" "$workdir" || _inside "$path" "$temp"; then - continue - fi - if _ancestor_of "$path" "$workdir" || _ancestor_of "$path" "$temp"; then - clean_dir "$path" "$workdir" "$temp" - continue - fi - rm -rf "$path" 2>/dev/null || true - done < <(find "$dir" -mindepth 1 -maxdepth 1 -print0 2>/dev/null) +# Empty dir $1 (/tmp), keeping the dir itself. +clean_tmp() { + find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null } -clean_tmp() { - local workdir="${RUNNER_WORKDIR:-/tmp/runner/work}" - local temp="${RUNNER_TEMP:-}" - workdir="${workdir%/}" - temp="${temp%/}" +cleanup() { + local before after + echo "[cleanup] Running post-job cleanup..." + before=$(df -h / | awk 'NR==2 {print $4}') - # Only a RUNNER_TEMP strictly under /tmp needs sparing. - case "$temp" in /tmp/*) ;; *) temp="" ;; esac + clean_tmp /tmp - clean_dir /tmp "$workdir" "$temp" -} + # Gradle daemon / journals / locks only. Keep modules-2, build-cache-*, + # transforms-*. $HOME covers both the root (default) and RUN_AS_ROOT=false + # (/home/runner) runner users. + rm -rf "$HOME"/.gradle/daemon/ "$HOME"/.gradle/caches/journal-* 2>/dev/null + # Shallow on purpose: Gradle's lock files sit at most three levels under + # caches/, and a find over the whole (multi-GB) tree is what made this slow. + find "$HOME"/.gradle/caches -maxdepth 3 -name '*.lock' -delete 2>/dev/null -clean_tmp + # Runner diagnostic logs + find /actions-runner/_diag -name '*.log' -delete 2>/dev/null -# Gradle daemon / journals / locks only. Keep modules-2, build-cache-*, -# transforms-*. $HOME covers both the root (default) and RUN_AS_ROOT=false -# (/home/runner) runner users. -rm -rf "$HOME"/.gradle/daemon/ 2>/dev/null || true -rm -rf "$HOME"/.gradle/caches/journal-* 2>/dev/null || true -# Shallow on purpose: Gradle's lock files sit at most three levels under -# caches/, and a find over the whole (multi-GB) tree is what made this slow. -find "$HOME"/.gradle/caches -maxdepth 3 -name '*.lock' -delete 2>/dev/null || true + # The job is over: the pruner may trim this slot again. + if [ -n "${RUNNER_SLOT_DIR:-}" ]; then rm -f "$RUNNER_SLOT_DIR/busy"; fi -# Runner diagnostic logs -find /actions-runner/_diag -name '*.log' -delete 2>/dev/null || true -find /runner-data/_diag -name '*.log' -delete 2>/dev/null || true + after=$(df -h / | awk 'NR==2 {print $4}') + echo "[cleanup] Post-job cleanup complete. Disk free: ${before} -> ${after}" +} -AFTER=$(df -h / | awk 'NR==2 {print $4}') -echo "[cleanup] Post-job cleanup complete. Disk free: ${BEFORE} -> ${AFTER}" +# Sourced (tests/): define the functions only. +[ "${BASH_SOURCE[0]}" = "$0" ] || return 0 +cleanup diff --git a/docker-compose.yml b/docker-compose.yml index 792cb11..12ad47d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,46 +1,63 @@ -# Three ephemeral runners, each with its own work and cache volumes, plus a -# `pruner` for maintenance. Never share a cache volume between runners: -# concurrent jobs would corrupt it. The action archive cache is the exception: -# read-only in every runner, filled by the pruner. `volumes:` lists replace on -# merge, so docker.sock is repeated per service. Small hosts can layer -# hosts/no-caches.compose.yml on top (see README). -# The environment is its own anchor so a service can change one key (LABELS) -# without restating the rest: a `<<:` merge replaces a whole mapping, so -# `environment:` in a service would otherwise drop every other variable. +# RUNNERS ephemeral runners, CONTROL_RUNNERS coordination runners and a +# `pruner` for maintenance. Size a host by setting RUNNERS in .env +# (hosts/sizing.sh suggests a value); there is nothing per runner to copy. +# +# State lives in one host directory, RUNNER_DATA_DIR, bind-mounted at the SAME +# path inside every container. On start each container claims a free slot, +# RUNNER_DATA_DIR/slots/, with a lock it holds for its whole life (see +# preflight-entrypoint.sh), and keeps its work dir and caches there: +# slots//work RUNNER_WORKDIR. The same path on the host, so the +# `docker run -v` / container actions a job hands +# to the host daemon see the checkout (issue #19). +# slots//cache/ bun, npm, pnpm, gradle, cargo, rustup, konan, +# mise, hk, pub and the hosted tool cache; the +# default home paths (/root/.bun, ...) link here. +# One slot per running container, so no two concurrent jobs share a cache. +# The action archive cache is the only shared state: read-only in the +# runners, filled by the pruner. x-runner-env: &runner-env - APP_ID: ${APP_ID} - APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} - APP_LOGIN: ${APP_LOGIN} - RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner} - RUNNER_WORKDIR: /tmp/runner/work - RUNNER_SCOPE: org - ORG_NAME: ${APP_LOGIN} - LABELS: ${LABELS:-docker} - RUNNER_GROUP: ${RUNNER_GROUP:-Default} - # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY - # non-empty value - including "false" - turns the flag on. - # - # EPHEMERAL: "true" is what these runners have always actually done: the old - # `EPHEMERAL: "false"` passed --ephemeral too. Each runner takes exactly one - # job, deregisters, exits, and `restart: unless-stopped` brings up a fresh - # container that registers again. That is intended: a clean container per - # job, while the named volumes below carry the caches across containers. - EPHEMERAL: "true" - # Empty (the default) leaves runner self-update on. Set it in .env to ANY - # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables - # them too. See .env.example. - DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} - # Ephemeral runners re-download every action archive for every job, and the - # few public IPs of these hosts hit codeload.github.com's 429 limit. The - # runner (>= 2.319) reads /_/.tar.gz before it - # downloads, and only reads: the `pruner` service fills the shared - # runner-action-cache volume (see pruner.sh), mounted read-only below. - ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE: /opt/actionarchivecache + APP_ID: ${APP_ID} + APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} + APP_LOGIN: ${APP_LOGIN} + RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner} + RUNNER_SCOPE: org + ORG_NAME: ${APP_LOGIN} + LABELS: ${LABELS:-docker} + RUNNER_GROUP: ${RUNNER_GROUP:-Default} + RUNNER_DATA_DIR: ${RUNNER_DATA_DIR:-/var/lib/github-runner} + RUNNER_CACHES: ${RUNNER_CACHES:-on} + # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY + # non-empty value - including "false" - turns the flag on. + # + # Each runner takes exactly one job, deregisters, exits, and + # `restart: unless-stopped` starts it again to register afresh. The slot's + # caches carry over; /tmp and the job's leftovers do not (cleanup.sh). + EPHEMERAL: "true" + # Empty (the default) leaves runner self-update on. Set it in .env to ANY + # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables + # them too. See .env.example. + DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} + # Ephemeral runners re-download every action archive for every job, and the + # few public IPs of these hosts hit codeload.github.com's 429 limit. The + # runner (>= 2.319) reads /_/.tar.gz before it + # downloads, and only reads: the `pruner` service fills the shared + # runner-action-cache volume (see pruner.sh), mounted read-only below. + ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE: /opt/actionarchivecache x-runner: &runner image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped + # .env is passed whole, so RUNNER__LABELS works for any slot id without + # listing each one here. Optional, so `docker compose config` runs without it. + env_file: + - path: .env + required: false environment: *runner-env + # Optional per-runner limits (hosts/sizing.sh suggests them; 0 = none). With + # a CPU limit the preflight sizes Gradle/cargo/make/cmake parallelism to it, + # so runners sharing a host don't each spawn a worker per host CPU. + cpus: ${RUNNER_CPUS:-0} + mem_limit: ${RUNNER_MEMORY:-0} # Runner.Listener is the long-lived process of a registered runner. Exec form # on purpose: under CMD-SHELL the wrapping `sh -c` carries the pattern in its # own command line and `pgrep -f` would match itself, healthy forever. The @@ -54,167 +71,58 @@ x-runner: &runner start_period: 120s volumes: - /var/run/docker.sock:/var/run/docker.sock + - ${RUNNER_DATA_DIR:-/var/lib/github-runner}:${RUNNER_DATA_DIR:-/var/lib/github-runner} + - runner-action-cache:/opt/actionarchivecache:ro security_opt: - label:disable -# Coordination runners carry only CONTROL_LABELS (default `control`), never a -# work label, so no build or test can land on them: they exist so the short -# jobs a merge waits on (a picker, a required summary) never queue behind long -# work. Those jobs read the API and check nothing out, so each service mounts -# no docker.sock and no tool cache, only its work directory and the read-only -# action cache. -x-control: &control - <<: *runner - profiles: [control] - environment: - <<: *runner-env - LABELS: ${CONTROL_LABELS:-control} - RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner}-control - services: - # Each runner registers with the host's LABELS, or with RUNNER__LABELS when - # that is set: one runner of a host can then join another pool (a `large` - # runner on a host whose others stay `light`) without a second host file. + # Slots 1..RUNNERS. Each registers with the host's LABELS, or with + # RUNNER__LABELS when that is set: one runner of a host can then join + # another pool (a `large` runner on a host whose others stay `light`). runner: <<: *runner - environment: - <<: *runner-env - LABELS: ${RUNNER_1_LABELS:-${LABELS:-docker}} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - runner-work:/tmp/runner/work - - runner-bun:/root/.bun - - runner-npm:/root/.npm - - runner-pnpm:/root/.cache/pnpm - - runner-gradle:/root/.gradle - - runner-cargo:/root/.cargo - - runner-rustup:/root/.rustup - - runner-konan:/root/.konan - - runner-mise-data:/root/.local/share/mise - - runner-mise-cache:/root/.cache/mise - - runner-hk:/root/.cache/hk - - runner-action-cache:/opt/actionarchivecache:ro - runner-2: - <<: *runner - environment: - <<: *runner-env - LABELS: ${RUNNER_2_LABELS:-${LABELS:-docker}} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-2:/tmp/runner/work - - runner-bun-2:/root/.bun - - runner-npm-2:/root/.npm - - runner-pnpm-2:/root/.cache/pnpm - - runner-gradle-2:/root/.gradle - - runner-cargo-2:/root/.cargo - - runner-rustup-2:/root/.rustup - - runner-konan-2:/root/.konan - - runner-mise-data-2:/root/.local/share/mise - - runner-mise-cache-2:/root/.cache/mise - - runner-hk-2:/root/.cache/hk - - runner-action-cache:/opt/actionarchivecache:ro - runner-3: + deploy: + replicas: ${RUNNERS:-3} + + # Coordination runners carry only CONTROL_LABELS (default `control`), never a + # work label, so no build or test can land on them: they exist so the short + # jobs a merge waits on (a picker, a required summary) never queue behind long + # work. Those jobs read the API and check nothing out, so they get no + # docker.sock and no tool caches (slots c1..cN, work dir only). + control: <<: *runner + deploy: + replicas: ${CONTROL_RUNNERS:-0} environment: <<: *runner-env - LABELS: ${RUNNER_3_LABELS:-${LABELS:-docker}} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-3:/tmp/runner/work - - runner-bun-3:/root/.bun - - runner-npm-3:/root/.npm - - runner-pnpm-3:/root/.cache/pnpm - - runner-gradle-3:/root/.gradle - - runner-cargo-3:/root/.cargo - - runner-rustup-3:/root/.rustup - - runner-konan-3:/root/.konan - - runner-mise-data-3:/root/.local/share/mise - - runner-mise-cache-3:/root/.cache/mise - - runner-hk-3:/root/.cache/hk - - runner-action-cache:/opt/actionarchivecache:ro - # Coordination runners (see x-control), off unless COMPOSE_PROFILES=control. - control: - <<: *control + LABELS: ${CONTROL_LABELS:-control} + RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner}-control + RUNNER_SLOT_PREFIX: c + RUNNER_CACHES: "off" volumes: - - runner-work-control:/tmp/runner/work + - ${RUNNER_DATA_DIR:-/var/lib/github-runner}:${RUNNER_DATA_DIR:-/var/lib/github-runner} - runner-action-cache:/opt/actionarchivecache:ro - control-2: - <<: *control - volumes: - - runner-work-control-2:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - # Maintenance off the job path (pruner.sh): nightly cache caps and docker - # prune, hourly action-cache fill. Mounts only what it caps. + + # Maintenance off the job path (pruner.sh): cache budgets per slot and docker + # prune, plus the hourly action-cache fill. pruner: image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped entrypoint: ["/usr/local/bin/pruner.sh"] + env_file: + - path: .env + required: false + # .env (env_file above) brings the App credentials it prefetches actions + # with and every pruner tunable. environment: - CACHES_ROOT: /caches - APP_ID: ${APP_ID} # App token for authenticated action downloads - APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} - APP_LOGIN: ${APP_LOGIN} - ACTION_CACHE_CAP_BYTES: ${ACTION_CACHE_CAP_BYTES:-2147483648} - PREFETCH_SOURCES: ${PREFETCH_SOURCES:-} - PREFETCH_EXTRA: ${PREFETCH_EXTRA:-} - PREFETCH_INTERVAL_SECONDS: ${PREFETCH_INTERVAL_SECONDS:-3600} - CACHE_CAP_BYTES: ${CACHE_CAP_BYTES:-21474836480} - PRUNE_INTERVAL_SECONDS: ${PRUNE_INTERVAL_SECONDS:-86400} + RUNNER_DATA_DIR: ${RUNNER_DATA_DIR:-/var/lib/github-runner} volumes: - /var/run/docker.sock:/var/run/docker.sock - - runner-gradle:/caches/1/gradle - - runner-bun:/caches/1/bun - - runner-mise-cache:/caches/1/mise-cache - - runner-hk:/caches/1/hk - - runner-mise-data:/caches/1/mise-data - - runner-gradle-2:/caches/2/gradle - - runner-bun-2:/caches/2/bun - - runner-mise-cache-2:/caches/2/mise-cache - - runner-hk-2:/caches/2/hk - - runner-mise-data-2:/caches/2/mise-data - - runner-gradle-3:/caches/3/gradle - - runner-bun-3:/caches/3/bun - - runner-mise-cache-3:/caches/3/mise-cache - - runner-hk-3:/caches/3/hk - - runner-mise-data-3:/caches/3/mise-data + - ${RUNNER_DATA_DIR:-/var/lib/github-runner}:${RUNNER_DATA_DIR:-/var/lib/github-runner} - runner-action-cache:/opt/actionarchivecache security_opt: - label:disable volumes: runner-action-cache: {} # read-only in runners, filled by the pruner - runner-work: {} - runner-work-2: {} - runner-work-3: {} - runner-work-control: {} - runner-work-control-2: {} - runner-bun: {} - runner-bun-2: {} - runner-bun-3: {} - runner-npm: {} - runner-npm-2: {} - runner-npm-3: {} - runner-pnpm: {} - runner-pnpm-2: {} - runner-pnpm-3: {} - runner-gradle: {} - runner-gradle-2: {} - runner-gradle-3: {} - runner-cargo: {} - runner-cargo-2: {} - runner-cargo-3: {} - runner-rustup: {} - runner-rustup-2: {} - runner-rustup-3: {} - runner-konan: {} - runner-konan-2: {} - runner-konan-3: {} - runner-mise-data: {} - runner-mise-data-2: {} - runner-mise-data-3: {} - runner-mise-cache: {} - runner-mise-cache-2: {} - runner-mise-cache-3: {} - runner-hk: {} - runner-hk-2: {} - runner-hk-3: {} diff --git a/hosts/no-caches.compose.yml b/hosts/no-caches.compose.yml deleted file mode 100644 index 4f7cc25..0000000 --- a/hosts/no-caches.compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -# For small hosts (e.g. ARM64 boards with little disk): no per-runner cache -# volumes, only docker.sock, the work volume and the action archive cache. -# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml (in .env) -# `!override` replaces the base volume lists instead of merging into them. -services: - runner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - runner-2: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-2:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - runner-3: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-3:/tmp/runner/work - - runner-action-cache:/opt/actionarchivecache:ro - pruner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-action-cache:/opt/actionarchivecache diff --git a/hosts/sizing.sh b/hosts/sizing.sh new file mode 100755 index 0000000..339256c --- /dev/null +++ b/hosts/sizing.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Suggest .env settings for this host. Read-only: it inspects the machine and +# prints a snippet to paste into .env next to docker-compose.yml. +# +# hosts/sizing.sh [light|large] +# +# Profile = what one runner should get: +# light 1 CPU, 3 GiB checks, lint, Next.js/bun builds (default) +# large 2 CPU, 6 GiB Gradle/Android and other heavy builds +# Override with RUNNER_CPUS / RUNNER_MEM_GIB. The host keeps HOST_RESERVE_GIB +# (default 2) of memory for itself and the docker daemon. +# +# Linux only (reads /proc and uses GNU df). Run it on the host, not in a +# runner container. +set -euo pipefail + +profile="${1:-light}" +case "$profile" in + light) cpus="${RUNNER_CPUS:-1}" mem="${RUNNER_MEM_GIB:-3}" ;; + large) cpus="${RUNNER_CPUS:-2}" mem="${RUNNER_MEM_GIB:-6}" ;; + *) echo "usage: $0 [light|large]" >&2; exit 2 ;; +esac +reserve="${HOST_RESERVE_GIB:-2}" +data_dir="${RUNNER_DATA_DIR:-/var/lib/github-runner}" + +nproc=$(nproc) +mem_gib=$(awk '/^MemTotal:/ {printf "%d", $2 / 1048576}' /proc/meminfo) + +# Size the disk that will hold the data dir (its nearest existing parent). +probe="$data_dir" +while [ ! -e "$probe" ]; do probe=$(dirname "$probe"); done +read -r disk_bytes avail_bytes < <(df -B1 --output=size,avail "$probe" | awk 'NR==2 {print $1, $2}') + +by_cpu=$((nproc / cpus)) +by_mem=$(((mem_gib - reserve) / mem)) +runners=$((by_cpu < by_mem ? by_cpu : by_mem)) +[ "$runners" -ge 1 ] || runners=1 + +# The pruner's own budget (CACHE_BUDGET_BYTES / CACHE_BUDGET_PCT, honoring +# them when set), sized against this disk. Printed so the per-runner share is +# visible. pruner.sh only defines functions when sourced. +# shellcheck source=SCRIPTDIR/../pruner.sh +source "$(dirname "$0")/../pruner.sh" +budget=$(RUNNER_DATA_DIR="$probe" slot_budget 1) +gib() { awk -v b="$1" 'BEGIN {printf "%.1f", b / 1073741824}'; } + +arch=$(uname -m) +kvm=no +if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then kvm=yes; fi +docker_version=$(docker version --format '{{.Server.Version}}' 2>/dev/null || echo unknown) + +cat < by cpu $by_cpu, by memory $by_mem +RUNNERS=$runners +RUNNER_DATA_DIR=$data_dir +# Per-runner limits keep one job from starving the others; the preflight then +# caps Gradle/cargo/make/cmake workers to RUNNER_CPUS. +RUNNER_CPUS=$cpus +RUNNER_MEMORY=${mem}g +# cache budget: $(gib "$budget") GiB for the host, ~$(gib $((budget / runners))) GiB per slot +EOF +case "$arch" in + aarch64|arm64) + echo "# arm64: the Android SDK's build tools are x86_64-only, so use the slim image" + echo "RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner:slim" ;; +esac +if [ "$runners" -eq 1 ] && [ "$budget" -lt $((10 * 1024 * 1024 * 1024)) ]; then + echo "# little disk: consider RUNNER_CACHES=off" +fi diff --git a/matrix.json b/matrix.json index bf1fe20..3b68981 100644 --- a/matrix.json +++ b/matrix.json @@ -2,11 +2,7 @@ "java_versions": [ "17", "21", - "22", - "23", - "24", "25", - "26", "27" ], "default_jdk": "27", diff --git a/pre-job.sh b/pre-job.sh index 3149263..a94b941 100755 --- a/pre-job.sh +++ b/pre-job.sh @@ -2,9 +2,14 @@ # Pre-job hook for GitHub Actions runner. # Invoked via ACTIONS_RUNNER_HOOK_JOB_STARTED before each job. # -# JOB_COMPLETED does not run when the runner process dies (OOM). This sweep -# covers that case: delete incomplete/poison entries only. Do not rm -rf -# cache trees — named volumes are the warm cache. +# 1. Mark the slot busy so pruner.sh leaves its caches alone while the job +# runs. If the pruner is mid-pass on this slot, wait for it (bounded) so a +# job never starts on a half-evicted cache. +# 2. Crash sweep. JOB_COMPLETED (cleanup.sh) does not run when the runner +# process dies (OOM), and a `busy` marker it never removed says so. Then, +# and only then, delete what a killed job leaves half-written. Do not rm -rf +# cache trees: the slot caches are the warm cache. Without a slot (the image +# run outside compose) there is no marker, so it always sweeps. # # Intentionally not using set -e: individual prune failures should not # prevent the job from starting. @@ -15,17 +20,35 @@ # exact cache key (no restore-keys) so a corrupt entry cannot be carried # forward across lockfile changes. -echo "[pre-job] Surgical prune of incomplete cache entries..." +PRUNE_WAIT_SECONDS="${PRUNE_WAIT_SECONDS:-600}" -# Bun: a killed job leaves partial tarballs (*.tmp / *.part). The next job -# then fails with "Fail extracting tarball for ..." and it looks like a -# dependency problem rather than a runner problem. -find "$HOME"/.bun/install/cache \( -name '*.tmp' -o -name '*.part' \) -delete 2>/dev/null || true +crashed=true +if [ -n "${RUNNER_SLOT_DIR:-}" ] && [ -d "$RUNNER_SLOT_DIR" ]; then + [ -e "$RUNNER_SLOT_DIR/busy" ] || crashed=false + if exec 8>"$RUNNER_SLOT_DIR/.prune.lock"; then + flock -w "$PRUNE_WAIT_SECONDS" 8 || + echo "[pre-job] pruner still busy on $RUNNER_SLOT_DIR after ${PRUNE_WAIT_SECONDS}s; starting anyway" + touch "$RUNNER_SLOT_DIR/busy" + exec 8>&- + fi +fi + +[ "$crashed" = true ] || exit 0 +echo "[pre-job] Previous job did not finish; sweeping incomplete cache entries..." + +# Bun: a killed job leaves partial extractions (*.tmp / *.part, and the .tmp +# scratch dir) next to the package folders: at the top of the cache, or one +# level down for @scope. The next job then fails with "Fail extracting tarball +# for ..." and it looks like a dependency problem rather than a runner +# problem. Only those two levels: the cache holds hundreds of thousands of +# files, and a file inside a package that happens to end in .tmp is content. +bun_cache="$HOME/.bun/install/cache" +find "$bun_cache" "$bun_cache"/@*/ -mindepth 1 -maxdepth 1 \( -name '*.tmp' -o -name '*.part' \) \ + -exec rm -rf {} + 2>/dev/null # Gradle: leftover lock / journal / daemon state from a killed compile. # Keep modules-2, build-cache-*, and transforms-*. -rm -f "$HOME"/.gradle/caches/modules-2/modules-2.lock 2>/dev/null || true -rm -rf "$HOME"/.gradle/caches/journal-* 2>/dev/null || true -rm -rf "$HOME"/.gradle/daemon/ 2>/dev/null || true +rm -f "$HOME"/.gradle/caches/modules-2/modules-2.lock 2>/dev/null +rm -rf "$HOME"/.gradle/caches/journal-* "$HOME"/.gradle/daemon/ 2>/dev/null -echo "[pre-job] Prune complete." +echo "[pre-job] Sweep complete." diff --git a/preflight-entrypoint.sh b/preflight-entrypoint.sh index 5f5b013..9fea950 100644 --- a/preflight-entrypoint.sh +++ b/preflight-entrypoint.sh @@ -1,7 +1,21 @@ #!/bin/bash -# Preflight wrapper around the upstream myoung34 entrypoint. +# Preflight wrapper around the upstream myoung34 entrypoint. Two jobs: # -# When CONFIGURED_ACTIONS_RUNNER_FILES_DIR is set, the upstream entrypoint +# 1. Slot claim (when RUNNER_DATA_DIR is set, as docker-compose.yml does). +# Every runner container of a host shares RUNNER_DATA_DIR, bind-mounted at +# the same path on host and container. This claims the lowest free +# slots/ with flock on its .lock and keeps the lock +# for the container's whole life (fd 9 survives `exec`), so a slot's work +# dir and caches are only ever used by one container at a time. Then: +# - RUNNER_WORKDIR = slots//work. It is the same path on the host, so +# `docker run -v "$PWD:..."` and container actions mount the checkout. +# - LABELS = RUNNER__LABELS when that is set (numeric slots only). +# - The tools' default cache paths link into slots//cache/, +# unless RUNNER_CACHES=off (small hosts, control runners). +# pre-job.sh / cleanup.sh mark the slot busy for pruner.sh through +# RUNNER_SLOT_DIR. +# +# 2. Registration check. When CONFIGURED_ACTIONS_RUNNER_FILES_DIR is set, the upstream entrypoint # copies the persisted config into /actions-runner and skips registration # whenever a .runner file exists — with no validation of the stored # credentials and no fallback to ACCESS_TOKEN. If the runner was meanwhile @@ -21,6 +35,125 @@ log() { echo "[preflight] $*"; } +# store:path pairs. The path is the tool's default location for root (jobs run +# as root, HOME=/root); pruner.sh knows how to evict each store safely. +CACHE_LINKS=( + bun:/root/.bun + npm:/root/.npm + pnpm/store:/root/.local/share/pnpm # store-dir default: one store, one unit + pnpm/cache:/root/.cache/pnpm # metadata cache + gradle:/root/.gradle + cargo:/root/.cargo + rustup:/root/.rustup + konan:/root/.konan + mise-data:/root/.local/share/mise + mise-cache:/root/.cache/mise + hk:/root/.cache/hk + pub:/root/.pub-cache + toolcache:/opt/hostedtoolcache # AGENT_TOOLSDIRECTORY: setup-* actions, Flutter +) + +# Claim the lowest free slot under $1 into SLOT_ID. fd 9 stays open on the +# claimed .lock in this shell, which then execs the runner with it. +claim_slot() { + local root="$1" prefix="${RUNNER_SLOT_PREFIX:-}" n=1 id + while :; do + id="${prefix}${n}" + mkdir -p "$root/slots/$id" || return 1 + exec 9>"$root/slots/$id/.lock" || return 1 + if flock -n 9; then + SLOT_ID="$id" + return 0 + fi + exec 9>&- + n=$((n + 1)) + done +} + +# Point each default cache path at the slot. Anything already at the path +# (the container's writable layer from an earlier run, or an empty image dir) +# is replaced: the slot is the cache. +# Restarts (every job) find the links in place and fork nothing. +link_caches() { + local cache="$1" pair store path + for pair in "${CACHE_LINKS[@]}"; do + store="${pair%%:*}" + path="${pair#*:}" + [ -L "$path" ] && [ "$path" -ef "$cache/$store" ] && continue + mkdir -p "$cache/$store" "${path%/*}" + rm -rf "$path" + ln -s "$cache/$store" "$path" + done +} + +# CPUs this container may use: the cgroup v2 quota (cpu.max, from compose +# `cpus:`) rounded up, or nothing when unlimited. `nproc` inside a container +# reports the host's CPUs whatever the quota, and the tools size their worker +# pools from it. +cpu_quota() { + local quota period + read -r quota period < /sys/fs/cgroup/cpu.max 2>/dev/null || return 1 + if [ "$quota" = max ] || [ "${period:-0}" -le 0 ]; then return 1; fi + echo $(((quota + period - 1) / period)) +} + +# Under a CPU limit, cap the build tools' parallelism to it, unless the job +# environment already chose. The JVM and Node read the cgroup themselves. +tune_parallelism() { + local n + n=$(cpu_quota) || return 0 + export RUNNER_CPU_LIMIT="$n" + export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$n}" + export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-$n}" + export MAKEFLAGS="${MAKEFLAGS:--j$n}" + case "${GRADLE_OPTS:-}" in + *org.gradle.workers.max*) ;; + *) export GRADLE_OPTS="${GRADLE_OPTS:+$GRADLE_OPTS }-Dorg.gradle.workers.max=$n" ;; + esac + log "cpu limit $n: build parallelism capped to $n" +} + +# Install rustup (no toolchain) into ~/.cargo, which is the slot's cargo cache +# when caches are on, from the image's rustup-init. Offline and done once per +# slot: later starts find it. Toolchains then persist in ~/.rustup. +ensure_rustup() { + [ -x /usr/local/bin/rustup-init ] || return 0 + [ -x "$HOME/.cargo/bin/rustup" ] && return 0 + # Self-update off: the image pins rustup, and the check is a network call on + # every toolchain install. + if rustup-init -y -q --no-modify-path --default-toolchain none >/dev/null 2>&1 && + "$HOME/.cargo/bin/rustup" set auto-self-update disable >/dev/null 2>&1; then + log "rustup installed into $HOME/.cargo" + else + log "rustup-init failed; jobs install rustup themselves" + fi +} + +setup_slot() { + local root="${RUNNER_DATA_DIR:-}" id dir labels_var + [ -n "$root" ] || return 0 + # Not in $(...): a subshell would close fd 9, and with it the claim. + if ! claim_slot "$root"; then + log "cannot claim a slot under $root/slots; running without one" + return 0 + fi + id="$SLOT_ID" + dir="$root/slots/$id" + mkdir -p "$dir/work" + export RUNNER_SLOT="$id" RUNNER_SLOT_DIR="$dir" RUNNER_WORKDIR="$dir/work" + export RUNNER_NAME_PREFIX="${RUNNER_NAME_PREFIX:-runner}-${id}" + labels_var="RUNNER_${id}_LABELS" + if [[ "$id" =~ ^[0-9]+$ ]] && [ -n "${!labels_var:-}" ]; then + export LABELS="${!labels_var}" + fi + if [ "${RUNNER_CACHES:-on}" = off ]; then + log "slot $id ($dir), caches off" + else + link_caches "$dir/cache" + log "slot $id ($dir), caches under $dir/cache" + fi +} + # The runner's IsConfigured()/HasCredentials() checks treat .runner_migrated / # .credentials_migrated as equivalent to the primary files, so a wipe must # remove all five or config.sh refuses to re-register while run.sh can't load @@ -118,5 +251,11 @@ preflight() { esac } +# Sourced (tests/): define the functions only. +[ "${BASH_SOURCE[0]}" = "$0" ] || return 0 + +setup_slot +ensure_rustup +tune_parallelism preflight exec /entrypoint.sh "$@" diff --git a/pruner.sh b/pruner.sh index 9dedb1b..17f0681 100755 --- a/pruner.sh +++ b/pruner.sh @@ -1,17 +1,26 @@ #!/bin/bash -# Nightly pruner. Run by the `pruner` compose service (same image, this script -# as the entrypoint), NOT by a runner job hook. +# Pruner. Run by the `pruner` compose service (same image, this script as the +# entrypoint), NOT by a runner job hook. # # These scans used to sit in cleanup.sh and ran after every job, costing # 21-39 s of runner time each time and freeing nothing (issue #16). Here they -# run once a day, off the job path: -# 1. LRU size caps on the per-runner cache volumes (mounted under -# $CACHES_ROOT//, see docker-compose.yml). Caches whose -# entries are only valid whole (bun packages, mise tool versions) evict -# whole entries; the rest evict single files. +# run off the job path: +# 1. A size budget per runner slot ($RUNNER_DATA_DIR/slots//cache/), +# enforced with one global LRU across all the stores of the slot. What is +# evictable is described by STORE_TABLE below: caches whose entries are only +# valid whole (bun packages, gradle versions, mise tool versions, ...) evict +# whole directories, the rest evict single files. # 2. docker system prune / builder prune on the host daemon (docker.sock). # 3. Hourly, in the background: fill the shared action archive cache. # +# Job safety: a slot is pruned only while holding its `.prune.lock` (pre-job.sh +# waits on it) and only when it is not mid-job: a `busy` marker younger than 6 h +# means a job is running there, so the slot is skipped; an older one is stale. +# +# Adaptive schedule: every PRUNE_CHECK_SECONDS a cheap df runs; a full pass +# happens when free space is below PRUNE_FREE_PCT, or when PRUNE_INTERVAL_SECONDS +# have passed since the last pass. Under disk pressure docker prune is tighter. +# # Action archive cache: the runner copies # $ACTION_CACHE_DIR/_/.tar.gz instead of downloading when it # exists, but never writes it. prefetch_actions() fills it as the GitHub App, so @@ -20,152 +29,303 @@ # `pruner.sh prefetch` runs it once (rollout warm-up). # # Env: -# PRUNE_INTERVAL_SECONDS sleep between runs (default 86400) -# PRUNE_INITIAL_DELAY sleep before the first run (default 300; keeps a +# RUNNER_DATA_DIR data root holding slots// (/var/lib/github-runner) +# CACHE_BUDGET_BYTES total cache budget over all slots (60 GiB) +# CACHE_BUDGET_PCT ... capped to this % of the filesystem holding +# RUNNER_DATA_DIR (40); the smaller wins and is split +# evenly between the slots +# PRUNE_CHECK_SECONDS df check period (3600) +# PRUNE_FREE_PCT full pass when free space is below (15) +# PRUNE_INTERVAL_SECONDS full pass at least this often (86400) +# PRUNE_INITIAL_DELAY sleep before the first check (300; keeps a # `docker compose up -d` from running a scan at once) -# PRUNE_RUN_ONCE=true run once and exit (manual / testing) -# CACHES_ROOT where the cache volumes are mounted (/caches) -# CACHE_CAP_BYTES cap for each file-level cache dir (20 GiB) -# MISE_DATA_CAP_BYTES cap for each mise installs tree (20 GiB) +# PRUNE_RUN_ONCE=true run one pass and exit (manual / testing) # ACTION_CACHE_CAP_BYTES cap for the action archive cache (2 GiB) # PREFETCH_SOURCES / PREFETCH_EXTRA "owner/repo@ref ..." to scan / fetch as-is # PREFETCH_INTERVAL_SECONDS prefetch period (default 3600; 0 = off) -# DOCKER_PRUNE_UNTIL / DOCKER_BUILDER_PRUNE_UNTIL (24h / 72h) +# DOCKER_PRUNE_UNTIL / DOCKER_PRUNE_UNTIL_LOW / DOCKER_BUILDER_PRUNE_UNTIL +# (24h / 2h when below PRUNE_FREE_PCT / 72h) # # Intentionally not using set -e: one failing step must not stop the others. -CACHES_ROOT="${CACHES_ROOT:-/caches}" -CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB -MISE_DATA_CAP_BYTES="${MISE_DATA_CAP_BYTES:-21474836480}" # 20 GiB +RUNNER_DATA_DIR="${RUNNER_DATA_DIR:-/var/lib/github-runner}" +CACHE_BUDGET_BYTES="${CACHE_BUDGET_BYTES:-64424509440}" # 60 GiB +CACHE_BUDGET_PCT="${CACHE_BUDGET_PCT:-40}" +PRUNE_CHECK_SECONDS="${PRUNE_CHECK_SECONDS:-3600}" +PRUNE_FREE_PCT="${PRUNE_FREE_PCT:-15}" +PRUNE_INTERVAL_SECONDS="${PRUNE_INTERVAL_SECONDS:-86400}" +BUSY_STALE_MINUTES=360 # a `busy` marker older than 6 h is a crashed job's leftover ACTION_CACHE_DIR=/opt/actionarchivecache # mount point in docker-compose.yml ACTION_CACHE_CAP_BYTES="${ACTION_CACHE_CAP_BYTES:-2147483648}" # 2 GiB PREFETCH_SOURCES="${PREFETCH_SOURCES:-}" PREFETCH_EXTRA="${PREFETCH_EXTRA:-}" GITHUB_API="https://api.github.com" DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" +DOCKER_PRUNE_UNTIL_LOW="${DOCKER_PRUNE_UNTIL_LOW:-2h}" DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } -dir_bytes() { - du -sb "$1" 2>/dev/null | awk '{print $1}' -} +# Evictable units per store, one rule per line: store|kind|glob|skip +# kind d: every directory matching glob (relative to the store dir, never a +# symlink) is one unit, removed whole; skip is a case pattern for the +# relative path of units to leave alone. Glob "." makes the whole +# store dir one unit (pnpm: store/ + cache/ are all-or-nothing); it is +# emptied, but its subdirs are kept, as container symlinks point there. +# kind f: every regular file named skip (a find -name pattern) under the dir +# glob is one unit. +# Anything no rule matches (bun's bin/ and @t@ piles, cargo's bin/ and registry +# index, rustup's settings.toml, ...) is never touched. Hooks for the few stores +# needing more than a rule are in unit_key / unit_protected / unit_pre_evict / +# unit_post_evict. Leftovers of killed bun extractions (*.tmp, *.part) are +# pre-job.sh's job: it sweeps them before the next job of a crashed slot. +STORE_TABLE=' +bun|d|install/cache/*@*|install/cache/@* +bun|d|install/cache/@*/*@*|install/cache/@t@/* +npm|f|_cacache|* +pnpm|d|.| +gradle|d|caches/modules-2/files-2.1/*/*/*| +gradle|d|caches/transforms-*/*| +gradle|d|wrapper/dists/*| +gradle|d|caches/jars-*/*| +gradle|f|caches/build-cache-1|* +cargo|f|registry/cache|*.crate +cargo|d|registry/src/*/*| +cargo|d|git/checkouts/*| +cargo|d|git/db/*| +rustup|d|toolchains/*| +konan|d|dependencies/*| +konan|d|kotlin-native-prebuilt-*| +mise-data|d|installs/*/*| +pub|d|hosted/*/*| +pub|d|git/*| +toolcache|d|*/*| +mise-cache|f|.|* +hk|f|.|* +actions|f|.|* +' -# Print how many bytes dir $1 is over cap $2. Returns 1 (printing nothing) when -# the dir is missing, unreadable, or already within the cap. -excess_bytes() { - local dir="$1" cap="$2" size - [ -d "$dir" ] || return 1 - size=$(dir_bytes "$dir") - [ -n "$size" ] || return 1 - if [ "$size" -le "$cap" ]; then - log "$dir ${size}B <= cap ${cap}B" >&2 - return 1 - fi - log "$dir is ${size}B (cap ${cap}B)" >&2 - echo $((size - cap)) +# Stores of a runner slot (everything but the shared action archive cache). +slot_stores() { + awk -F'|' 'NF > 3 && $1 != "actions" && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" } -# LRU-evict regular files, least recently accessed first, until $1 is at or -# under $2 bytes. Ordered by atime (a read counts as use; Gradle never touches -# mtime on read, so mtime order evicted the most-used dependencies first). -# Volumes mounted relatime still refresh atime on the first read after a -# change. Does not wholesale wipe the tree. -lru_cap() { - local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" - local excess freed=0 rec fsize path - excess=$(excess_bytes "$dir" "$cap") || return 0 - log "evicting least recently used files from $dir" - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - rec="${rec#* }" # drop atime - fsize="${rec%% *}" - path="${rec#* }" - if [ -z "$path" ] || [ -z "$fsize" ]; then continue; fi - rm -f "$path" 2>/dev/null || true - freed=$((freed + fsize)) - done < <(find "$dir" -type f -printf '%A@\t%s\t%p\0' 2>/dev/null | sort -z -n) - find "$dir" -type d -empty -delete 2>/dev/null || true - log "$dir freed ~${freed}B" +dir_bytes() { + du -sb "$1" 2>/dev/null | awk '{printf "%.0f\n", $1}' } -# mise installs (installs//) are only useful whole: deleting -# single files would leave a tool that exists but is broken. Evict whole -# / directories instead, oldest install first, until the -# installs tree fits under the cap. -lru_cap_mise_installs() { - local data="$1" cap="${2:-$MISE_DATA_CAP_BYTES}" - local installs="$data/installs" - local excess freed=0 rec path dsize - excess=$(excess_bytes "$installs" "$cap") || return 0 - log "evicting oldest tool versions from $installs" - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - path="${rec#* }" - [ -d "$path" ] || continue - dsize=$(dir_bytes "$path") - rm -rf "$path" 2>/dev/null || true - freed=$((freed + ${dsize:-0})) - done < <(find "$installs" -mindepth 2 -maxdepth 2 -type d -printf '%T@\t%p\0' 2>/dev/null | sort -z -n) - log "$installs freed ~${freed}B" +# default_toolchain from rustup's settings.toml ($1 = RUSTUP_HOME). +rustup_default() { + sed -n 's/^default_toolchain[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1/settings.toml" 2>/dev/null | head -n 1 } # bun treats install/cache/@@@@/ as complete whenever the folder # exists, so a file evicted from it is silently missing from every later # node_modules (next@16.3.8 lost dist/compiled/server-only/empty.js, breaking # every Next build). 0-byte files went first: copying them never reads, so -# their atime never moved. Evict whole package folders instead, least recently -# used first by the atime of their package.json (read on every install and -# import), until the whole volume ($1, mounted from /root/.bun) fits under the -# cap. Never touches $1/bin (the bun binary itself) or the @t@ manifest piles. -lru_cap_bun() { - local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" - local cache="$dir/install/cache" - local excess freed=0 rec path dsize base t - [ -d "$cache" ] || return 0 - # Leftovers of interrupted extractions; the age guard spares live installs. - find "$cache" -mindepth 1 -maxdepth 2 ! -name .tmp \( -name '*.tmp' -o -name '*.part' \) \ - -mmin +60 -exec rm -rf {} + 2>/dev/null - find "$cache/.tmp" -mindepth 1 -maxdepth 1 -mmin +60 -exec rm -rf {} + 2>/dev/null - excess=$(excess_bytes "$dir" "$cap") || return 0 - log "evicting least recently used packages from $cache" +# their atime never moved. Hence whole packages, least recently used first by +# the atime of their package.json (read on every install and import). + +# LRU key override for dir unit $2 of store $1; prints nothing to keep the +# newest-atime-of-any-file key. +unit_key() { + [ "$1" = bun ] || return 0 + stat -c %X "$2/package.json" 2>/dev/null || stat -c %Y "$2" 2>/dev/null +} + +# Succeeds when unit $3 (relative to dir $2 of store $1) must not be evicted. +unit_protected() { + local def + [ "$1" = rustup ] || return 1 + def=$(rustup_default "$2") + [ -n "$def" ] || return 1 + case "${3##*/}" in + "$def" | "$def"-*) return 0 ;; # `stable` is the dir stable- + esac + return 1 +} + +# toolcache: /.complete says / is whole. Drop the markers +# first so an interrupted removal never leaves a "complete" half tree. +unit_pre_evict() { + [ "$1" = toolcache ] || return 0 + find "$2" -mindepth 1 -maxdepth 1 -name '*.complete' -delete 2>/dev/null +} + +# After unit $2 of store $1 is gone. +unit_post_evict() { + local base + [ "$1" = bun ] || return 0 + # Drop the /@@@ symlink pointing at it, and the dir if empty. + base="${2##*/}" + rm -f "${2%/*}/${base%%@*}/${base#*@}" 2>/dev/null + rmdir "${2%/*}/${base%%@*}" 2>/dev/null + return 0 +} + +# Print NUL-terminated "keysizestorestoppath" records, one +# per evictable unit of store $1 living in dir $2. key is the LRU order (atime, +# seconds); stop is the dir above which emptied parents are not removed. +collect_units() { + local store="$1" sd="$2" s kind glob skip pre stop + [ -d "$sd" ] || return 0 + while IFS='|' read -r s kind glob skip; do + [ "$s" = "$store" ] || continue + if [ "$kind" = f ]; then + [ -d "$sd/$glob" ] || continue + find "$sd/$glob" -type f -name "$skip" -printf "%A@\t%s\t$store\t-\t%p\0" 2>/dev/null + continue + fi + pre="${glob%%[*?[]*}" + case "$pre" in + */*) pre="${pre%/*}" ;; + *) pre="" ;; + esac + stop="$sd${pre:+/$pre}" + ( + cd "$sd" 2>/dev/null || exit 0 + shopt -s nullglob + # shellcheck disable=SC2086 # glob expansion is the point + for rel in $glob; do + if [ ! -d "$rel" ] || [ -L "$rel" ]; then continue; fi + # shellcheck disable=SC2254 # skip is a pattern + case "$rel" in $skip) continue ;; esac + if unit_protected "$store" "$sd" "$rel"; then continue; fi + stats=$(find "$rel" -printf '%y\t%A@\t%s\n' 2>/dev/null | + awk -F'\t' '{ s += $3; if ($1 == "f" && $2 + 0 > m) m = $2 + 0 } END { printf "%.6f\t%.0f\n", m, s }') + key="${stats%%$'\t'*}" + t=$(unit_key "$store" "$rel") + [ -z "$t" ] || key="$t" + printf '%s\t%s\t%s\t%s\t%s\0' "$key" "${stats#*$'\t'}" "$store" "$stop" "$sd/$rel" + done + ) + done <<<"$STORE_TABLE" +} + +# Remove unit $3 of store $1 (stop dir $2): a whole directory (then its emptied +# parents up to $2), or a single file. +evict_unit() { + local store="$1" stop="$2" path="$3" p + if [ "${path##*/}" = . ]; then + find "${path%/.}" -mindepth 2 -delete 2>/dev/null + find "${path%/.}" -mindepth 1 -maxdepth 1 ! -type d -delete 2>/dev/null + elif [ -d "$path" ] && [ ! -L "$path" ]; then + unit_pre_evict "$store" "$path" + rm -rf "$path" 2>/dev/null + unit_post_evict "$store" "$path" + p="${path%/*}" + while [ "$p" != "$stop" ] && [ "${p#"$stop"/}" != "$p" ]; do + rmdir "$p" 2>/dev/null || break + p="${p%/*}" + done + else + rm -f "$path" 2>/dev/null + fi +} + +# Keep the stores given as = pairs ($2...) within $1 bytes in total, +# evicting units across all of them in global LRU order, least recently used +# first (a read counts as use; Gradle never touches mtime on read, so mtime +# order evicted the most-used dependencies first). Volumes mounted relatime +# still refresh atime on the first read after a change. +lru_entries() { + local budget="$1" spec store sd total=0 sz excess freed=0 n=0 + local rec key size stop path + local -A dirs=() touched=() + shift + for spec in "$@"; do + store="${spec%%=*}" + sd="${spec#*=}" + [ -d "$sd" ] || continue + dirs[$store]="$sd" + sz=$(dir_bytes "$sd") + total=$((total + ${sz:-0})) + done + if [ "$total" -le "$budget" ]; then + log "${total}B <= budget ${budget}B" + return 0 + fi + excess=$((total - budget)) + log "${total}B (budget ${budget}B), evicting least recently used entries" while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - path="${rec#* }" - [ -d "$path" ] || continue - dsize=$(dir_bytes "$path") - rm -rf "$path" 2>/dev/null || true - freed=$((freed + ${dsize:-0})) - # Drop the /@@@ symlink pointing at it, and the dir if empty. - base="${path##*/}" - rm -f "${path%/*}/${base%%@*}/${base#*@}" 2>/dev/null - rmdir "${path%/*}/${base%%@*}" 2>/dev/null + if [ "$freed" -ge "$excess" ]; then break; fi + IFS=$'\t' read -r key size store stop path <<<"$rec" + if [ -z "$path" ] || [ -z "$size" ]; then continue; fi + evict_unit "$store" "$stop" "$path" + freed=$((freed + size)) + n=$((n + 1)) + touched[$store]=1 done < <( - # @... at the top level, @scope/@... one level down. - find "$cache" -mindepth 1 -maxdepth 2 -type d -name '*@*' ! -name '@*' \ - \( -path "$cache/@*/*" -o ! -path "$cache/*/*" \) -print0 2>/dev/null | - while IFS= read -r -d '' path; do - t=$(stat -c %X "$path/package.json" 2>/dev/null || stat -c %Y "$path" 2>/dev/null) || continue - printf '%s\t%s\0' "$t" "$path" - done | sort -z -n + for store in "${!dirs[@]}"; do collect_units "$store" "${dirs[$store]}"; done | + LC_ALL=C sort -z -n ) - find "$cache" -mindepth 1 -maxdepth 1 -name '@*' ! -name '@t@' -type d -empty -delete 2>/dev/null || true - log "$cache freed ~${freed}B" + # Emptied directories under file-level stores. + for store in "${!touched[@]}"; do + while IFS='|' read -r s kind glob _; do + if [ "$s" = "$store" ] && [ "$kind" = f ] && [ -d "${dirs[$store]}/$glob" ]; then + find "${dirs[$store]}/$glob" -mindepth 1 -type d -empty -delete 2>/dev/null + fi + done <<<"$STORE_TABLE" + done + if [ "$freed" -lt "$excess" ]; then + log "evicted $n entries, ~${freed}B freed; still ~$((excess - freed))B over budget (protected or untracked data)" + else + log "evicted $n entries, ~${freed}B freed" + fi +} + +# Total cache budget in bytes of one slot, with $1 slots sharing it. +slot_budget() { + local slots="$1" fs total + total="$CACHE_BUDGET_BYTES" + fs=$(df -B1 --output=size "$RUNNER_DATA_DIR" 2>/dev/null | awk 'NR == 2 {print $1}') + if [ -n "$fs" ] && [ $((fs * CACHE_BUDGET_PCT / 100)) -lt "$total" ]; then + total=$((fs * CACHE_BUDGET_PCT / 100)) + fi + echo $((total / slots)) +} + +# Free space of the filesystem holding RUNNER_DATA_DIR, in whole percent. +free_pct() { + local dir="$RUNNER_DATA_DIR" + [ -d "$dir" ] || dir=/ + df -B1 --output=avail,size "$dir" 2>/dev/null | awk 'NR == 2 && $2 > 0 {print int($1 * 100 / $2)}' +} + +# Prune one slot ($1 = slot dir) down to $2 bytes, unless a job is running in +# it. The flock keeps jobs from starting meanwhile: pre-job.sh waits on it. +prune_slot() { + local slot="$1" budget="$2" id start="$SECONDS" + id="${slot##*/}" + ( + flock -n 9 || { log "slot $id: another prune holds the lock; skipping"; exit 0; } + if [ -e "$slot/busy" ]; then + if [ -n "$(find "$slot/busy" -maxdepth 0 -mmin "-$BUSY_STALE_MINUTES" 2>/dev/null)" ]; then + log "slot $id: job running (busy marker); skipping" + exit 0 + fi + log "slot $id: stale busy marker (> ${BUSY_STALE_MINUTES} min); pruning anyway" + fi + args=() + for store in $(slot_stores); do args+=("$store=$slot/cache/$store"); done + log "slot $id: budget ${budget}B" + lru_entries "$budget" "${args[@]}" + ) 9>"$slot/.prune.lock" + log "slot $id: took $((SECONDS - start))s" } prune_caches() { - local runner - for runner in "$CACHES_ROOT"/*/; do - [ -d "$runner" ] || continue - runner="${runner%/}" - log "capping caches under $runner" - lru_cap "$runner/gradle" - lru_cap_bun "$runner/bun" - lru_cap "$runner/mise-cache" - lru_cap "$runner/hk" # configs/.json + pklr/packages/.zip: whole files - lru_cap_mise_installs "$runner/mise-data" + local caches=() cache budget + for cache in "$RUNNER_DATA_DIR"/slots/*/cache; do + [ -d "$cache" ] && caches+=("$cache") done - lru_cap "$ACTION_CACHE_DIR" "$ACTION_CACHE_CAP_BYTES" + if [ "${#caches[@]}" -gt 0 ]; then + budget=$(slot_budget "${#caches[@]}") + for cache in "${caches[@]}"; do prune_slot "${cache%/cache}" "$budget"; done + else + log "no slots under $RUNNER_DATA_DIR/slots" + fi + lru_entries "$ACTION_CACHE_CAP_BYTES" "actions=$ACTION_CACHE_DIR" } gh_curl() { @@ -199,7 +359,8 @@ prefetch_one() { fi file="$dir/$sha.tar.gz" [ -s "$file" ] && return 0 - mkdir -p "$dir" && tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 + mkdir -p "$dir" || return 1 + tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 if gh_curl -o "$tmp" "$GITHUB_API/repos/$name/tarball/$sha" && tar -tzf "$tmp" >/dev/null 2>&1; then chmod 644 "$tmp" && mv -f "$tmp" "$file" && { log "prefetch: cached $name@$sha ($ref)" >&2; echo 1; return 0; } fi @@ -253,13 +414,16 @@ prefetch_actions() { GH_TOKEN="" } +# $1 = low: free space is below PRUNE_FREE_PCT, so prune docker harder. prune_docker() { + local until="$DOCKER_PRUNE_UNTIL" + [ "${1:-}" = low ] && until="$DOCKER_PRUNE_UNTIL_LOW" if ! docker info >/dev/null 2>&1; then log "docker daemon unreachable; skipping docker prune" return 0 fi - log "docker system prune -af --filter until=${DOCKER_PRUNE_UNTIL}" - docker system prune -af --filter "until=${DOCKER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' + log "docker system prune -af --filter until=${until}" + docker system prune -af --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' log "docker builder prune -af --filter until=${DOCKER_BUILDER_PRUNE_UNTIL}" docker builder prune -af --filter "until=${DOCKER_BUILDER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' } @@ -268,7 +432,7 @@ run_once() { local before after before=$(df -h / | awk 'NR==2 {print $4}') prune_caches - prune_docker + prune_docker "${1:-}" after=$(df -h / | awk 'NR==2 {print $4}') log "done. Disk free: ${before} -> ${after}" } @@ -297,7 +461,20 @@ if [ "${PREFETCH_INTERVAL_SECONDS:-3600}" -gt 0 ]; then fi sleep "${PRUNE_INITIAL_DELAY:-300}" +last=0 while true; do - run_once - sleep "${PRUNE_INTERVAL_SECONDS:-86400}" + free=$(free_pct) + low="" + now=$(date +%s) + if [ -n "$free" ] && [ "$free" -lt "$PRUNE_FREE_PCT" ]; then + low=low + log "free space ${free}% < ${PRUNE_FREE_PCT}%: full pass" + elif [ $((now - last)) -ge "$PRUNE_INTERVAL_SECONDS" ]; then + log "free space ${free:-?}%, ${PRUNE_INTERVAL_SECONDS}s since the last pass: full pass" + fi + if [ -n "$low" ] || [ $((now - last)) -ge "$PRUNE_INTERVAL_SECONDS" ]; then + run_once "$low" + last=$(date +%s) + fi + sleep "$PRUNE_CHECK_SECONDS" done diff --git a/tests/image-smoke.sh b/tests/image-smoke.sh new file mode 100755 index 0000000..d00ee6d --- /dev/null +++ b/tests/image-smoke.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Smoke test for a built runner image: the toolchain a job relies on must be +# present and runnable, with the entrypoint bypassed so no runner registers. +# bash tests/image-smoke.sh [expected-jdk-major] [expected-node-major] [std|ndk|slim] +# Variants: std (runner) and ndk (runner-ndk) must carry the Android SDK; slim +# (runner-slim) must carry java/node/tools but no Android SDK. +# The check snippets are single-quoted on purpose: they expand inside the image. +# shellcheck disable=SC2016 +set -euo pipefail + +image="${1:?usage: tests/image-smoke.sh [jdk-major] [node-major] [std|ndk|slim]}" +want_jdk="${2:-}" +want_node="${3:-}" +variant="${4:-std}" +case "$variant" in + std | ndk | slim) ;; + *) echo "unknown variant '$variant' (want std, ndk or slim)" >&2; exit 2 ;; +esac + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } +pass() { echo "ok: $*"; } +indent() { printf ' %s\n' "${1//$'\n'/$'\n '}"; } + +# in_image