benchmarkoor-run #12767
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: benchmarkoor-run | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| clients: | |
| description: "JSON array of execution clients to benchmark" | |
| required: false | |
| default: '["besu", "erigon", "geth", "nethermind", "reth", "ethrex"]' | |
| instance-id: | |
| description: "Set this if you have multiple configurations for the same client but just want to run one of them" | |
| required: false | |
| default: "" | |
| snapshot: | |
| description: "Snapshot path" | |
| required: false | |
| type: choice | |
| options: | |
| - "state-actor/v1" | |
| - "jochemnet/v1" | |
| context: | |
| description: "Test context" | |
| required: false | |
| type: choice | |
| options: | |
| - "repricing" | |
| default: "repricing" | |
| subdir: | |
| description: "Subdirectory" | |
| required: true | |
| type: choice | |
| options: | |
| # State actor | |
| - "v1/bal-devnet-7" | |
| - "v1/glamsterdam-devnet-6" | |
| - "v1/glamsterdam-devnet-7" | |
| - "v1/glamsterdam-devnet-7-full" | |
| # Jochemnet | |
| - "jochemnet/v1/glamsterdam-devnet-7" | |
| test-type: | |
| description: "Test type to run" | |
| required: false | |
| type: choice | |
| options: | |
| - "stateful" | |
| - "compute" | |
| default: "stateful" | |
| timeouts: | |
| description: 'Per-phase timeouts (minutes) as JSON — build: state-actor build step, run: benchmark run step, global: whole-job cap' | |
| required: false | |
| default: '{"build": 360, "run": 480, "global": 840}' | |
| image: | |
| description: "Benchmarkoor docker image" | |
| required: false | |
| default: "ghcr.io/ethpandaops/benchmarkoor:master" | |
| git-ref: | |
| description: "Git reference to checkout for benchmarkoor" | |
| required: false | |
| default: "" | |
| run-config: | |
| description: 'Raw YAML config content to pass via --config (appended after URL config)' | |
| required: false | |
| default: '' | |
| jobs: | |
| benchmarkoor: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client: ${{ fromJSON(inputs.clients) }} | |
| runs-on: | |
| group: gasbenchmark | |
| labels: | |
| # The runner pool follows the subdir, because a run can only happen where | |
| # its datadirs live: jochemnet subdirs need the jochemnet hosts, every | |
| # other subdir stays on the v1 pool. Mirrors benchmarkoor-build.yaml. | |
| - >- | |
| ${{ | |
| contains(inputs.subdir, 'jochemnet') | |
| && format('benchmarkoor-v1-jochemnet-runner-{0}', matrix.client) | |
| || format('benchmarkoor-v1-runner-{0}', matrix.client) | |
| }} | |
| # Whole-job cap; each phase also has its own step-level timeout below | |
| # (both read from the timeouts JSON input). Keep global >= build + run. | |
| timeout-minutes: ${{ fromJSON(inputs.timeouts).global }} | |
| steps: | |
| - name: "Infos: Client=${{ matrix.client }} | Snapshot=${{ inputs.snapshot }} | TestType=${{ inputs.test-type }} | Context=${{ inputs.context }} | Subdirectory=${{ inputs.subdir }}" | |
| run: | | |
| echo "client: ${{ matrix.client }}" | |
| echo "snapshot: ${{ inputs.snapshot }}" | |
| echo "test-type: ${{ inputs.test-type }}" | |
| echo "subdir: ${{ inputs.subdir }}" | |
| echo "timeouts: ${{ inputs.timeouts }}" | |
| echo "image: ${{ inputs.image }}" | |
| echo "git-ref: ${{ inputs.git-ref }}" | |
| - uses: actions/checkout@v5 | |
| - name: Calculate timeouts | |
| id: calc | |
| run: | | |
| # benchmarkoor's internal run timeout, 30m under each phase's step timeout. | |
| echo "builder_timeout=$(( ${{ fromJSON(inputs.timeouts).build }} - 30 ))m" >> "$GITHUB_OUTPUT" | |
| echo "runner_timeout=$(( ${{ fromJSON(inputs.timeouts).run }} - 30 ))m" >> "$GITHUB_OUTPUT" | |
| # Only for snapshots state_actor synthesises. jochemnet is an existing | |
| # network whose datadir is already on the host, so its datadirs | |
| # builder.yaml declares no builders and this step fails outright with | |
| # "no builders configured; nothing to build". | |
| - name: Build state-actor data directory | |
| if: ${{ !contains(inputs.snapshot, 'jochemnet') }} | |
| timeout-minutes: ${{ fromJSON(inputs.timeouts).build }} | |
| uses: ethpandaops/benchmarkoor@master | |
| env: | |
| BENCHMARKOOR_BUILDER_RUN_TIMEOUT: ${{ steps.calc.outputs.builder_timeout }} | |
| with: | |
| mode: build | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| image: ${{ inputs.image }} | |
| git-ref: ${{ inputs.git-ref }} | |
| upload-artifacts: "false" | |
| build-config-urls: >- | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/global.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/datadirs/${{ inputs.snapshot }}/global.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/datadirs/${{ inputs.snapshot }}/builder.yaml, | |
| build-args: >- | |
| --rebuild-on-diff | |
| --limit-state-actor-target=${{ matrix.client }} | |
| - name: Run benchmarkoor | |
| timeout-minutes: ${{ fromJSON(inputs.timeouts).run }} | |
| uses: ethpandaops/benchmarkoor@master | |
| env: | |
| S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} | |
| API_INGEST_TOKEN: ${{ secrets.API_INGEST_TOKEN }} | |
| BENCHMARKOOR_RUNNER_RUN_TIMEOUT: ${{ steps.calc.outputs.runner_timeout }} | |
| with: | |
| mode: run | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| image: ${{ inputs.image }} | |
| git-ref: ${{ inputs.git-ref }} | |
| upload-artifacts: "false" | |
| run-config-urls: >- | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/global.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/resource-limits-eip-7870-fullnode.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/s3-upload.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/datadirs/${{ inputs.snapshot }}/global.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/datadirs/${{ inputs.snapshot }}/runner.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/contexts/${{ inputs.context }}/${{ inputs.subdir }}/global.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/contexts/${{ inputs.context }}/${{ inputs.subdir }}/test-source.${{ inputs.test-type }}.runner.yaml, | |
| https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs/contexts/${{ inputs.context }}/${{ inputs.subdir }}/clients.yaml | |
| run-args: >- | |
| --limit-instance-client=${{ matrix.client }} | |
| --limit-instance-id=${{ inputs.instance-id }} | |
| run-config: ${{ inputs.run-config }} |