Skip to content

Commit 29dc675

Browse files
anarazelanarazel
committed
WIP: measure resource usage during CI
1 parent bc2f523 commit 29dc675

2 files changed

Lines changed: 323 additions & 6 deletions

File tree

‎.github/workflows/pg-ci.yml‎

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -336,12 +336,31 @@ jobs:
336336
127.0.0.3 pg-loadbalancetest
337337
EOF
338338
339+
- &install_python_deps_step
340+
name: Install Python Deps
341+
shell: bash
342+
run: |
343+
where python || type python || true
344+
where python3 || type python3 || true
345+
where pip || type pip || true
346+
where pip3 || type pip3 || true
347+
348+
pip3 install --break-system-packages matplotlib psutil
349+
350+
python3 -c "import psutil; print(psutil.cpu_times())"
351+
352+
echo $(pwd)/src/tools/ci >> "$GITHUB_PATH"
353+
354+
339355
# By using a shell that includes su, the run commands themselves get
340356
# simpler. As there are quite a few commands that need to use su...
341357
- name: Configure
342358
shell: &su_postgres_shell |
343359
su postgres -c "bash --noprofile --norc -eo pipefail {0}"
344360
run: |
361+
env
362+
echo "$PATH"
363+
resmon.py --help
345364
meson setup \
346365
--buildtype=debug \
347366
--auto-features=disabled \
@@ -352,7 +371,7 @@ jobs:
352371
- name: Build
353372
shell: *su_postgres_shell
354373
run: &ninja_build_cmd |
355-
ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
374+
resmon.py --prefix resource-build --interval 0.5 ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
356375
ninja -C build -t missingdeps
357376

358377
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,12 +424,40 @@ jobs:
405424
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
406425
echo ::endgroup::
407426

408-
meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
427+
resmon.py --prefix resource-test --interval 0.5 meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} || exit 1
409428

410429
echo "Template use: "
411430
grep -r 'initializing database' build/meson-logs/testlog.txt|sort|uniq -c
412431
413432
433+
- &upload_resource_graph_test_step
434+
name: Upload Resource Graph Test
435+
uses: actions/upload-artifact@v7
436+
with:
437+
name: resource-test-${{env.GITHUB_JOB}}.png
438+
path: resource-test.png
439+
if-no-files-found: error
440+
archive: false
441+
442+
- &upload_resource_graph_build_step
443+
name: Upload Resource Graph Build
444+
uses: actions/upload-artifact@v7
445+
with:
446+
name: resource-build-${{env.GITHUB_JOB}}.png
447+
path: resource-build.png
448+
if-no-files-found: error
449+
archive: false
450+
451+
- &upload_resource_info_step
452+
name: Upload Resource Info
453+
uses: actions/upload-artifact@v7
454+
with:
455+
name: resource-info-${{env.GITHUB_JOB}}
456+
path: |
457+
resource-*.png
458+
resource-*.csv
459+
if-no-files-found: error
460+
414461
- &linux_collect_cores_step
415462
name: Core backtraces
416463
if: failure() && !cancelled()
@@ -500,6 +547,7 @@ jobs:
500547
- *ccache_restore_default_step
501548
- *ccache_restore_branch_step
502549
- *linux_prepare_workspace_step
550+
- *install_python_deps_step
503551

504552
- name: Configure
505553
shell: *su_postgres_shell
@@ -515,15 +563,21 @@ jobs:
515563
- name: Build
516564
shell: *su_postgres_shell
517565
run: |
518-
make -s -j${BUILD_JOBS} world-bin
566+
resmon.py --prefix resource-build --interval 0.5 \
567+
make -s -j${BUILD_JOBS} world-bin
519568
520569
- *ccache_decide_save_step
521570
- *ccache_save_step
522571

523572
- name: Test world
524573
shell: *su_postgres_shell
525574
run: |
526-
make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
575+
resmon.py --prefix resource-test --interval 0.5 \
576+
make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
577+
578+
- *upload_resource_graph_test_step
579+
- *upload_resource_graph_build_step
580+
- *upload_resource_info_step
527581

528582
- *linux_collect_cores_step
529583
- *upload_logs_step
@@ -562,6 +616,7 @@ jobs:
562616
- *ccache_restore_default_step
563617
- *ccache_restore_branch_step
564618
- *linux_prepare_workspace_step
619+
- *install_python_deps_step
565620

566621
- name: Configure
567622
shell: *su_postgres_shell
@@ -615,10 +670,22 @@ jobs:
615670
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
616671
617672
# Run the tests supporting running against an already running
618-
meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
673+
resmon.py --prefix resource-test-running --interval 0.5 \
674+
meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
619675
620676
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
621677
678+
- name: Upload Resource Graph Running
679+
uses: actions/upload-artifact@v7
680+
with:
681+
name: resource-test-running-${{env.GITHUB_JOB}}.png
682+
path: resource-test-running.png
683+
if-no-files-found: error
684+
archive: false
685+
- *upload_resource_graph_test_step
686+
- *upload_resource_graph_build_step
687+
- *upload_resource_info_step
688+
622689
- *linux_collect_cores_step
623690
- *upload_logs_step
624691

@@ -652,6 +719,7 @@ jobs:
652719
- *ccache_restore_default_step
653720
- *ccache_restore_branch_step
654721
- *linux_prepare_workspace_step
722+
- *install_python_deps_step
655723

656724
- name: Configure
657725
shell: *su_postgres_shell
@@ -674,6 +742,10 @@ jobs:
674742
shell: *su_postgres_shell
675743
run: *meson_test_world_cmd
676744

745+
- *upload_resource_graph_test_step
746+
- *upload_resource_graph_build_step
747+
- *upload_resource_info_step
748+
677749
- *linux_collect_cores_step
678750
- *upload_logs_step
679751

@@ -715,6 +787,7 @@ jobs:
715787
p5.34-io-tty
716788
p5.34-ipc-run
717789
python312
790+
py312-pip
718791
tcl
719792
zstd
720793
@@ -783,6 +856,7 @@ jobs:
783856
sh src/tools/ci/ci_macports_packages.sh $MACOS_PACKAGE_LIST
784857
# system python doesn't provide headers
785858
sudo /opt/local/bin/port select python3 python312
859+
sudo /opt/local/bin/port select pip3 pip312
786860
# Make macports install visible to subsequent steps
787861
echo /opt/local/sbin >> "$GITHUB_PATH"
788862
echo /opt/local/bin >> "$GITHUB_PATH"
@@ -801,6 +875,7 @@ jobs:
801875

802876
- *ccache_restore_default_step
803877
- *ccache_restore_branch_step
878+
- *install_python_deps_step
804879

805880
- name: Configure
806881
env:
@@ -828,6 +903,10 @@ jobs:
828903
ADDITIONAL_SETUP: ulimit -n 1024
829904
run: *meson_test_world_cmd
830905

906+
- *upload_resource_graph_test_step
907+
- *upload_resource_graph_build_step
908+
- *upload_resource_info_step
909+
831910
- name: Core backtraces
832911
if: failure() && !cancelled()
833912
run: src/tools/ci/cores_backtrace.sh macos "$HOME/cores"
@@ -998,6 +1077,7 @@ jobs:
9981077
name: Setup Windows debugger
9991078
shell: pwsh
10001079
run: src/tools/ci/gha_setup_windows_debugger.ps1
1080+
- *install_python_deps_step
10011081

10021082
- name: Configure
10031083
run: |
@@ -1014,7 +1094,7 @@ jobs:
10141094
- name: Build
10151095
run: |
10161096
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
1017-
ninja -C build ${{env.MBUILD_TARGET}} || exit 1
1097+
resmon.py --prefix resource-test --interval 0.5 ninja -C build ${{env.MBUILD_TARGET}} || exit 1
10181098
ninja -C build -t missingdeps
10191099
10201100
- name: Test world
@@ -1028,6 +1108,10 @@ jobs:
10281108
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
10291109
run: *meson_test_world_cmd
10301110

1111+
- *upload_resource_graph_test_step
1112+
- *upload_resource_graph_build_step
1113+
- *upload_resource_info_step
1114+
10311115
- *upload_logs_step
10321116

10331117

@@ -1101,6 +1185,7 @@ jobs:
11011185
${MINGW_PACKAGE_PREFIX}-make \
11021186
${MINGW_PACKAGE_PREFIX}-meson \
11031187
${MINGW_PACKAGE_PREFIX}-perl \
1188+
${MINGW_PACKAGE_PREFIX}-python-pip \
11041189
${MINGW_PACKAGE_PREFIX}-pkgconf \
11051190
${MINGW_PACKAGE_PREFIX}-readline \
11061191
${MINGW_PACKAGE_PREFIX}-zlib \
@@ -1126,6 +1211,7 @@ jobs:
11261211

11271212
- *ccache_restore_default_step
11281213
- *ccache_restore_branch_step
1214+
- *install_python_deps_step
11291215

11301216
- name: Configure
11311217
run: |
@@ -1146,6 +1232,10 @@ jobs:
11461232
- name: Test world
11471233
run: *meson_test_world_cmd
11481234

1235+
- *upload_resource_graph_test_step
1236+
- *upload_resource_graph_build_step
1237+
- *upload_resource_info_step
1238+
11491239
- *upload_logs_step
11501240

11511241

0 commit comments

Comments
 (0)