Skip to content

Commit 18d019f

Browse files
aduh95aduh95
committed
tools: use lighter shell to aggregate benchmark results
Signed-off-by: Antoine du Hamel <duhamelantoine1995@gmail.com>
1 parent ede6d6a commit 18d019f

4 files changed

Lines changed: 217 additions & 215 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 32 additions & 211 deletions
Original file line numberDiff line numberDiff line change
@@ -1,196 +1,27 @@
11
name: Benchmark
22

33
on:
4-
workflow_dispatch:
5-
inputs:
6-
repo:
7-
type: string
8-
description: GitHub repository to fetch from (default to the current repo)
9-
pr_id:
10-
type: number
11-
required: true
12-
description: The PR to test
13-
commit:
14-
required: true
15-
type: string
16-
description: The expect HEAD of the PR
17-
category:
18-
required: true
19-
type: string
20-
description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
21-
filter:
22-
type: string
23-
description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
24-
runs:
25-
type: number
26-
default: 30
27-
description: How many times to repeat each benchmark
28-
post-comment:
29-
type: boolean
30-
description: Post a comment linking to the run, with the aggregated result once known.
31-
token:
32-
type: string
33-
description: A GitHub token to post comments cross repository (not recommended)
4+
push:
345

356
permissions:
367
contents: read
378

389
jobs:
39-
post-comment:
40-
if: inputs.post-comment
41-
outputs:
42-
body: ${{ steps.comment.outputs.COMMENT_BODY }}
43-
url: ${{ steps.comment.outputs.COMMENT_URL }}
44-
runs-on: ubuntu-slim
45-
permissions:
46-
pull-requests: write
47-
steps:
48-
- name: Mark token input as sensitive
49-
if: inputs.token != ''
50-
run: echo "::add-mask::${{ inputs.token }}"
51-
- name: Add link to the current run
52-
id: comment
53-
run: |
54-
FILTER_IF_SET=
55-
[ -z "$FILTER" ] || FILTER_IF_SET=" / $FILTER"
56-
COMMENT_BODY="Benchmark GHA (${CATEGORIES}${FILTER_IF_SET}): ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
57-
echo "COMMENT_BODY=$COMMENT_BODY" >> "$GITHUB_OUTPUT"
58-
echo "COMMENT_URL=$(gh pr comment -R "${REPO:-$GITHUB_REPOSITORY}" "$PR_ID" --body "$COMMENT_BODY")" >> "$GITHUB_OUTPUT"
59-
env:
60-
CATEGORIES: ${{ inputs.category }}
61-
FILTER: ${{ inputs.filter }}
62-
GH_TOKEN: ${{ inputs.token || github.token }}
63-
REPO: ${{ inputs.repo }}
64-
PR_ID: ${{ inputs.pr_id }}
65-
build:
66-
strategy:
67-
fail-fast: true
68-
matrix:
69-
include:
70-
- runner: ubuntu-24.04
71-
system: x86_64-linux
72-
- runner: ubuntu-24.04-arm
73-
system: aarch64-linux
74-
- runner: macos-15-intel
75-
system: x86_64-darwin
76-
- runner: macos-latest
77-
system: aarch64-darwin
78-
name: '${{ matrix.system }}: with shared libraries'
79-
runs-on: ${{ matrix.runner }}
80-
steps:
81-
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
82-
with:
83-
repository: ${{ inputs.repo || github.repository }}
84-
ref: refs/pull/${{ inputs.pr_id }}/merge
85-
persist-credentials: false
86-
fetch-depth: 2
87-
88-
- name: Validate PR head and roll back to base commit
89-
run: |
90-
[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
91-
git reset HEAD^ --hard
92-
env:
93-
EXPECTED_SHA: ${{ inputs.commit }}
94-
95-
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
96-
with:
97-
extra_nix_config: sandbox = true
98-
99-
- uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
100-
with:
101-
# We do not pass any `authToken` to avoid polluting the cache with potentially untrusted code.
102-
name: nodejs
103-
104-
- name: Configure sccache
105-
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
106-
with:
107-
script: |
108-
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
109-
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
110-
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
111-
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
112-
113-
- name: Build Node.js on the base commit
114-
run: |
115-
nix-shell \
116-
-I nixpkgs=./tools/nix/pkgs.nix \
117-
--pure --keep TAR_DIR --keep FLAKY_TESTS \
118-
--keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
119-
--arg useSeparateDerivationForV8 true \
120-
--arg loadJSBuiltinsDynamically false \
121-
--arg ccache "${NIX_SCCACHE:-null}" \
122-
--arg devTools '[]' \
123-
--arg benchmarkTools '[]' \
124-
--run '
125-
make build-ci -j4 V=1
126-
'
127-
mv out/Release/node base_node
128-
129-
- name: Checkout the merge commit
130-
run: git reset FETCH_HEAD --hard
131-
132-
- name: Re-build Node.js on the merge commit
133-
# ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
134-
run: |
135-
nix-shell \
136-
-I nixpkgs=./tools/nix/pkgs.nix \
137-
--pure \
138-
--arg useSeparateDerivationForV8 true \
139-
--arg loadJSBuiltinsDynamically false \
140-
--arg ccache 'null' \
141-
--arg devTools '[]' \
142-
--arg benchmarkTools '[]' \
143-
--run '
144-
make -j4 V=1
145-
'
146-
147-
- name: Run benchmark
148-
run: |
149-
nix-shell \
150-
-I nixpkgs=./tools/nix/pkgs.nix \
151-
--pure --keep FILTER --keep LC_ALL --keep LANG \
152-
--arg loadJSBuiltinsDynamically false \
153-
--arg ccache 'null' \
154-
--arg icu 'null' \
155-
--arg sharedLibDeps '{}' \
156-
--arg devTools '[]' \
157-
--run '
158-
set -o pipefail
159-
./base_node benchmark/compare.js \
160-
--filter "$FILTER" \
161-
--runs ${{ inputs.runs }} \
162-
--old ./base_node --new ./node \
163-
-- ${{ inputs.category }} \
164-
| tee /dev/stderr \
165-
> ${{ matrix.system }}.csv
166-
echo "> [!WARNING] "
167-
echo "> Do not take GHA benchmark results as face value, always confirm them"
168-
echo "> using a dedicated machine, e.g. Jenkins CI."
169-
echo
170-
echo "Benchmark results:"
171-
echo
172-
echo '"'"'```'"'"'
173-
Rscript benchmark/compare.R < ${{ matrix.system }}.csv
174-
echo '"'"'```'"'"'
175-
echo
176-
echo "> [!WARNING] "
177-
echo "> Do not take GHA benchmark results as face value, always confirm them"
178-
echo "> using a dedicated machine, e.g. Jenkins CI."
179-
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
180-
env:
181-
FILTER: ${{ inputs.filter }}
182-
183-
- name: Upload raw benchmark results
184-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
185-
with:
186-
name: csv-${{ matrix.system }}
187-
path: ${{ matrix.system }}.csv
188-
18910
aggregate-results:
190-
needs: [build, post-comment]
191-
if: ${{ always() && needs.build.result == 'success' && needs.post-comment.result == (inputs.post-comment && 'success' || 'skipped') }}
19211
name: Aggregate benchmark results
193-
runs-on: ubuntu-slim
12+
# strategy:
13+
# fail-fast: false
14+
# matrix:
15+
# include:
16+
# - runner: ubuntu-24.04
17+
# system: x86_64-linux
18+
# - runner: ubuntu-24.04-arm
19+
# system: aarch64-linux
20+
# # - runner: macos-15-intel
21+
# # system: x86_64-darwin
22+
# - runner: macos-latest
23+
# system: aarch64-darwin
24+
runs-on: ubuntu-latest # slim # ${{ matrix.runner }}
19425
permissions:
19526
contents: read
19627
pull-requests: write
@@ -199,6 +30,7 @@ jobs:
19930
with:
20031
persist-credentials: false
20132
sparse-checkout: |
33+
why.sh
20234
benchmark/*.R
20335
tools/nix/*.nix
20436
*.nix
@@ -210,17 +42,32 @@ jobs:
21042
pattern: csv-*
21143
merge-multiple: true
21244
path: raw-results
45+
repository: nodejs/node
46+
github-token: ${{ secrets.GH_USER_TOKEN }}
47+
run-id: 29243127788
21348

21449
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
21550
with:
21651
extra_nix_config: sandbox = true
52+
# extra_nix_config: |
53+
# sandbox = true
54+
# trusted-public-keys = nodejs-test.cachix.org-1:WvIzKH8IKBPG+VBNgA33RHVP11pL42HiqYSfvLYjwBc= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
55+
# substituters = https://nodejs-test.cachix.org https://cache.nixos.org/
56+
57+
- uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
58+
with:
59+
name: nodejs-test
60+
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
21761

21862
- name: Benchmark results
21963
run: |
22064
nix-shell \
22165
-I nixpkgs=./tools/nix/pkgs.nix \
22266
--pure \
223-
-E '(import <nixpkgs> {}).mkShell { buildInputs = import ./tools/nix/benchmarkTools.nix { withHttpBenchmarkDeps = false; }; }' \
67+
-E '(import <nixpkgs> {}).mkShellNoCC {
68+
buildInputs = import ./tools/nix/benchmarkTools.nix {
69+
withHttpBenchmarkDeps = false;
70+
}; }' \
22471
--run '
22572
export LC_ALL=C.UTF-8
22673
echo "> [!WARNING] "
@@ -236,30 +83,4 @@ jobs:
23683
echo "> [!WARNING] "
23784
echo "> Do not take GHA benchmark results as face value, always confirm them"
23885
echo "> using a dedicated machine, e.g. Jenkins CI."
239-
' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY"
240-
- name: Mark token input as sensitive
241-
if: inputs.token != ''
242-
run: echo "::add-mask::${{ inputs.token }}"
243-
- name: Edit comment
244-
if: inputs.post-comment
245-
run: |
246-
{
247-
echo "$COMMENT_BODY"
248-
echo
249-
echo '<details><summary>Results</summary>'
250-
echo
251-
cat body.txt
252-
echo
253-
echo '</details>'
254-
} | jq -Rrcs '{ body: . }' | gh api \
255-
--method PATCH \
256-
-H "Accept: application/vnd.github+json" \
257-
-H "X-GitHub-Api-Version: 2026-03-10" \
258-
"/repos/${REPO:-$GITHUB_REPOSITORY}/issues/comments/${COMMENT_URL##*-}" \
259-
--input -
260-
env:
261-
GH_TOKEN: ${{ inputs.token || github.token }}
262-
REPO: ${{ inputs.repo }}
263-
COMMENT_BODY: ${{ needs.post-comment.outputs.body }}
264-
COMMENT_URL: ${{ needs.post-comment.outputs.url }}
265-
PR_ID: ${{ inputs.pr_id }}
86+
' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY"

0 commit comments

Comments
 (0)