eval-tutorial.yml

name: Eval tutorial

on:
  pull_request:
    paths:
      - ".github/workflows/eval-tutorial.yml"
      - "evals/write-an-eval-that-can-say-no/**"
  push:
    branches: [main]
    paths:
      - ".github/workflows/eval-tutorial.yml"
      - "evals/write-an-eval-that-can-say-no/**"

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  run-example:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11", "3.12"]
    env:
      UV_PYTHON: ${{ matrix.python-version }}
    defaults:
      run:
        working-directory: evals/write-an-eval-that-can-say-no
    steps:
      - uses: actions/checkout@v6
      - uses: astral-sh/setup-uv@v8.1.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install locked dependencies
        run: |
          uv sync --locked
          uv run python --version
      - name: Regenerate frozen rows and check expected hash
        run: |
          uv run python freeze.py
          git diff --exit-code -- evals/FROZEN.sha256
      - name: Train models
        run: uv run python train.py
      - name: Test gate logic
        run: uv run python -m unittest test_gate
      - name: Candidate must be rejected
        run: |
          set +e
          uv run python run_eval.py candidate
          result=$?
          test "$result" -eq 1
      - name: Broken control must be rejected
        run: |
          set +e
          uv run python run_eval.py broken
          result=$?
          test "$result" -eq 1
      - name: Altered inputs must return exit 2
        run: |
          cp evals/gate.toml gate.toml.bak
          printf '\n' >> evals/gate.toml
          set +e
          uv run python run_eval.py candidate
          result=$?
          set -e
          mv gate.toml.bak evals/gate.toml
          test "$result" -eq 2
      - name: Missing model must return exit 3
        run: |
          mv candidate.joblib candidate.joblib.bak
          set +e
          uv run python run_eval.py candidate
          result=$?
          set -e
          mv candidate.joblib.bak candidate.joblib
          test "$result" -eq 3