Score submissions from R2 (private → aggregate) #54
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Score submissions from R2 (private → aggregate) | |
| # The scoring worker for the CLI→serverless→R2 intake. Maintainer-triggered (a human gate | |
| # on what becomes public — pred scoring itself is free, but we don't want raw spam auto- | |
| # published). It: | |
| # 1. pulls pending submissions (incoming/) + prior scored detail (results/) from private R2, | |
| # 2. re-verifies every certificate with pred (zero-trust) + the provenance gate, in a | |
| # network-isolated container, | |
| # 3. publishes ONLY the aggregate leaderboard to the public repo (site/results.json), | |
| # guarded so no certificate / rule identity can leak, | |
| # 4. keeps the scored detail (with certs) private — back in R2, never in git. | |
| # | |
| # Required repo secrets (Settings → Secrets → Actions), from a Cloudflare R2 "Object Read & | |
| # Write" API token: R2_ACCOUNT_ID R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_BUCKET | |
| # See intake/cloudflare-worker/README.md. | |
| on: | |
| workflow_dispatch: # manual trigger (maintainer) | |
| schedule: | |
| - cron: "0 16 * * *" # daily sweep of R2 for new submissions (16:00 UTC). | |
| permissions: | |
| contents: write # push the aggregate to a bot branch | |
| pull-requests: write # open the PR that updates the public leaderboard | |
| concurrency: | |
| group: score-from-r2 | |
| cancel-in-progress: false | |
| jobs: | |
| score: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Read benchmark version | |
| id: version | |
| run: echo "value=$(cat VERSION)" >> "$GITHUB_OUTPUT" | |
| # buildx with the docker-container driver — required for the type=gha build cache | |
| # below (the default docker driver can't export cache). | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Build scoring image (pred + verifier) | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| target: runtime | |
| build-args: PR_REF=${{ steps.version.outputs.value }} | |
| load: true | |
| tags: prb-scoring:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Pull pending submissions + prior scored detail from R2 | |
| id: pull | |
| env: | |
| R2_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| BUCKET: ${{ secrets.R2_BUCKET }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: auto | |
| run: | | |
| mkdir -p incoming results/scored | |
| # Snapshot the exact queue keys before downloading. Only this immutable manifest | |
| # may be archived later; an object arriving while scoring remains in incoming/. | |
| aws s3api list-objects-v2 --bucket "$BUCKET" --prefix incoming/ \ | |
| --endpoint-url "$R2_ENDPOINT" --query 'Contents[].Key' --output json \ | |
| | jq -r '.[]? | select(endswith(".json"))' > queue-manifest.txt | |
| while IFS= read -r key; do | |
| [ -n "$key" ] || continue | |
| rel="${key#incoming/}" | |
| [ "$rel" != "$key" ] && [ -n "$rel" ] || continue | |
| mkdir -p "incoming/$(dirname "$rel")" | |
| aws s3 cp "s3://$BUCKET/$key" "incoming/$rel" --endpoint-url "$R2_ENDPOINT" | |
| done < queue-manifest.txt | |
| # results/ = all prior scored detail, so the rebuilt board reflects full history. | |
| aws s3 sync "s3://$BUCKET/results" ./results/scored --endpoint-url "$R2_ENDPOINT" | |
| echo "pending snapshot: $(wc -l < queue-manifest.txt) object(s)" | |
| - name: Re-verify with pred (zero-trust + provenance), rebuild the leaderboard | |
| id: score | |
| continue-on-error: true | |
| run: | | |
| docker run --rm --network none \ | |
| -v "$PWD/incoming:/app/submissions" \ | |
| -v "$PWD/results:/app/results" \ | |
| --entrypoint python prb-scoring:latest \ | |
| -m benchmark.backend_score --official \ | |
| --local /app/submissions /app/results/scored | |
| - name: Persist results and transition only the snapshotted queue objects | |
| if: always() && steps.pull.outcome == 'success' | |
| env: | |
| R2_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| BUCKET: ${{ secrets.R2_BUCKET }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: auto | |
| run: | | |
| # Scored detail carries certificates — it goes back to PRIVATE R2, never to git. | |
| aws s3 sync ./results/scored "s3://$BUCKET/results" --endpoint-url "$R2_ENDPOINT" | |
| # FINISHED objects move to processed/. Permanent input failures move to failed/ | |
| # with their diagnostic status. Retryable or unclassified failures stay queued. | |
| python -m benchmark.r2_queue \ | |
| --manifest queue-manifest.txt --incoming-dir incoming > queue-transitions.tsv | |
| while IFS=$'\t' read -r source destination status_path status_destination; do | |
| [ -n "$source" ] || continue | |
| if [ "$status_destination" != "-" ]; then | |
| aws s3 cp "$status_path" "s3://$BUCKET/$status_destination" \ | |
| --endpoint-url "$R2_ENDPOINT" | |
| fi | |
| aws s3 mv "s3://$BUCKET/$source" "s3://$BUCKET/$destination" \ | |
| --endpoint-url "$R2_ENDPOINT" | |
| done < queue-transitions.tsv | |
| - name: Open ONE PR per new/changed submission entry (main is protected) | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # backend_score wrote one PUBLIC entry file per non-test submission into | |
| # results/scored/board/<slug>.json (slug = model--time--id). Each becomes its own | |
| # PR that adds site/results/<slug>.json — independently reviewed, merged, reverted. | |
| # The slug is deterministic, so re-scoring the same submission force-updates the SAME | |
| # branch/PR (idempotent — no duplicate PRs); a different submission is its own PR. | |
| shopt -s nullglob | |
| git config user.name "prb-bot" | |
| git config user.email "prb-bot@users.noreply.github.com" | |
| BASE="$(git rev-parse HEAD)" | |
| mkdir -p site/results | |
| opened=0 | |
| for f in results/scored/board/*.json; do | |
| slug="$(basename "$f" .json)" | |
| dest="site/results/${slug}.json" | |
| # Skip if this exact entry is already published on the base branch. | |
| if git cat-file -e "$BASE:$dest" 2>/dev/null && git show "$BASE:$dest" | cmp -s - "$f"; then | |
| echo "unchanged, skip: $slug"; continue | |
| fi | |
| MODEL="$(jq -r .model "$f")"; BUGS="$(jq -r '.bugs_found // 0' "$f")" | |
| TS="$(jq -r '.timestamp // "?"' "$f")" | |
| TITLE="leaderboard: ${MODEL} — ${BUGS} bug(s) @ ${TS}" | |
| BR="bot/leaderboard/${slug}" | |
| git checkout -B "$BR" "$BASE" >/dev/null | |
| cp "$f" "$dest" | |
| python .github/scripts/check_aggregate.py "$dest" # no certs / rule identities | |
| git add "$dest" | |
| git commit -q -m "$TITLE" | |
| git push -f origin "$BR" | |
| EXISTING="$(gh pr list --head "$BR" --state open --json number -q '.[0].number')" | |
| if [ -n "$EXISTING" ]; then | |
| gh pr edit "$EXISTING" --title "$TITLE" | |
| echo "updated PR #$EXISTING ($slug)" | |
| else | |
| gh pr create --base "${{ github.ref_name }}" --head "$BR" --title "$TITLE" \ | |
| --body "One submission's public entry (\`$dest\`) — aggregate only (counts / tokens / efficiency), no certificates or rule identities (guarded by \`check_aggregate.py\`). The full certificate stays private in R2. Merging rebuilds \`site/results.json\` from all entries and deploys via publish-on-merge. This PR is scoped to a single submission; merge/close/revert it independently." | |
| fi | |
| opened=$((opened+1)) | |
| done | |
| echo "opened/updated $opened submission PR(s)" | |
| git checkout -q "$BASE" 2>/dev/null || true | |
| - name: Report retryable scoring failure | |
| if: always() && steps.score.outcome == 'failure' | |
| run: | | |
| echo "One or more submissions had a retryable verifier/infrastructure failure." | |
| echo "They remain under R2 incoming/ and will be retried on the next run." | |
| exit 1 |