diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..428b0296 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,300 @@ +name: Benchmark + +# Counts the instructions of calls from Python into C++ (benchmarks/) under +# callgrind, on Linux x86_64 with a Release build of everything. +# - push to main: publishes the counts to gh-pages, as charts and as the +# benchmarks/baseline.json that pull requests compare against. +# - pull_request: counts the PR (merged into main) and compares it with +# main's published counts; fails when a benchmark runs more than THRESHOLD +# percent more instructions. Counts depend on the toolchain, so main is +# counted again on this runner when its published counts came from another +# toolchain (see the fingerprint), or to confirm a regression. + +on: + push: + branches: [main] + pull_request: + paths-ignore: + - 'README.md' + - 'docs/**' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: benchmark-${{ github.event.pull_request.number || github.ref }} + # PR-only: let every push to main reach gh-pages, one at a time. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # percent; a benchmark running more instructions than main by more fails + THRESHOLD: '5' + CPPINTEROP_EXTRA_INTERPRETER_ARGS: '-std=c++20' + +jobs: + benchmark: + name: benchmark/ubuntu-24.04/llvm22/py3.14 + runs-on: ubuntu-24.04 + steps: + # On a PR this is the merge commit: the PR as it would land on main. + - name: Checkout head + uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-python@v7 + with: + python-version: '3.14' + + # valgrind from conda-forge, as in the test suite's valgrind cell; it + # needs glibc's debug symbols from libc6-dbg + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends cmake ninja-build g++ libc6-dbg + + - uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1 + with: + environment-name: valgrind-env + create-args: valgrind + condarc: | + channels: + - conda-forge + + - name: Add valgrind to PATH + run: | + prefix="${MAMBA_ROOT_PREFIX:-$HOME/micromamba}/envs/valgrind-env" + echo "${prefix}/bin" >> "${GITHUB_PATH}" + # the benchmark library's callgrind markers need its headers + echo "CPPFLAGS=-I${prefix}/include" >> "${GITHUB_ENV}" + + - uses: compiler-research/ci-workflows/actions/setup-llvm@main + with: + version: '22' + os: ubuntu-24.04 + + # Everything besides the code under test that changes the counts. Counts + # are compared only within one fingerprint. + - name: Fingerprint the toolchain + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/bench" + # LLVM_DIR is /lib/cmake/llvm + llvm_config="${LLVM_DIR}/../../../bin/llvm-config" + { + echo "${ImageOS}" + ldd --version | head -n1 + g++ --version | head -n1 + echo "LLVM $("${llvm_config}" --version 2>/dev/null || echo "${LLVM_DIR}")" + python -c 'import sys; print("Python", sys.version.split()[0])' + valgrind --version + } | paste -sd ';' - > "${RUNNER_TEMP}/bench/fingerprint.txt" + cat "${RUNNER_TEMP}/bench/fingerprint.txt" + + - name: Build head + run: | + set -euo pipefail + export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" + python -m venv "${RUNNER_TEMP}/venv-head" + "${RUNNER_TEMP}/venv-head/bin/pip" install pytest + "${RUNNER_TEMP}/venv-head/bin/pip" install . \ + --config-settings=cmake.define.LLVM_DIR="${LLVM_DIR}" + + - name: Count head + env: + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + PYTHON="${RUNNER_TEMP}/venv-head/bin/python" benchmarks/callgrind.sh \ + "${RUNNER_TEMP}/bench/head.json" --commit "${COMMIT}" \ + --fingerprint "$(cat "${RUNNER_TEMP}/bench/fingerprint.txt")" + + # main's published counts serve when they share this run's fingerprint + # and show no regression; otherwise main is counted here. + - name: Fetch main's counts + id: stored + if: ${{ github.event_name == 'pull_request' }} + run: | + set -uo pipefail + out="${RUNNER_TEMP}/bench" + recount() { + echo "recount=true" >> "${GITHUB_OUTPUT}" + echo "note=Counted \`main\` on this runner: $1." >> "${GITHUB_OUTPUT}" + exit 0 + } + git fetch --depth=1 origin gh-pages \ + && git show FETCH_HEAD:benchmarks/baseline.json > "${out}/base.json" \ + || recount "it has no published counts" + python - "${out}/base.json" "${out}/head.json" <<'EOF' || recount "its published counts come from another toolchain" + import json, sys + base, head = (json.load(open(p)) for p in sys.argv[1:]) + sys.exit(base.get("fingerprint") != head["fingerprint"]) + EOF + python benchmarks/report.py compare --base "${out}/base.json" \ + --head "${out}/head.json" --threshold "${THRESHOLD}" > /dev/null \ + || recount "confirming a regression against its published counts" + echo "recount=false" >> "${GITHUB_OUTPUT}" + + # head's benchmarks, so that both builds run the same cases + - name: Checkout main + if: ${{ steps.stored.outputs.recount == 'true' }} + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.ref }} + path: base + persist-credentials: false + + - name: Build main + if: ${{ steps.stored.outputs.recount == 'true' }} + run: | + set -euo pipefail + export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" + python -m venv "${RUNNER_TEMP}/venv-base" + "${RUNNER_TEMP}/venv-base/bin/pip" install pytest + "${RUNNER_TEMP}/venv-base/bin/pip" install ./base \ + --config-settings=cmake.define.LLVM_DIR="${LLVM_DIR}" + + - name: Count main + if: ${{ steps.stored.outputs.recount == 'true' }} + run: | + set -euo pipefail + PYTHON="${RUNNER_TEMP}/venv-base/bin/python" benchmarks/callgrind.sh \ + "${RUNNER_TEMP}/bench/base.json" --commit "$(git -C base rev-parse HEAD)" \ + --fingerprint "$(cat "${RUNNER_TEMP}/bench/fingerprint.txt")" + + - name: Compare with main + if: ${{ github.event_name == 'pull_request' }} + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + NOTE: ${{ steps.stored.outputs.note }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + CHARTS_URL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/benchmarks/ + run: | + set -uo pipefail + out="${RUNNER_TEMP}/bench" + python benchmarks/report.py compare \ + --base "${out}/base.json" --head "${out}/head.json" \ + --threshold "${THRESHOLD}" \ + --base-name "${BASE_REF}" --head-name "#${PR_NUMBER}" \ + --repo-url "${REPO_URL}" --charts-url "${CHARTS_URL}" \ + ${NOTE:+--note "${NOTE}"} \ + --markdown "${out}/comment.md" + RETCODE=$? + cat "${out}/comment.md" >> "${GITHUB_STEP_SUMMARY}" + exit "${RETCODE}" + + - name: Export for the charts + if: ${{ github.event_name != 'pull_request' }} + run: | + out="${RUNNER_TEMP}/bench" + python benchmarks/report.py export "${out}/head.json" \ + --first-call "${out}/first_call.json" \ + --successive-call "${out}/successive_call.json" + + - name: Upload results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: benchmark-results + path: ${{ runner.temp }}/bench/ + if-no-files-found: warn + + report: + if: ${{ always() && github.event_name == 'pull_request' }} + needs: benchmark + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/download-artifact@v8 + with: + name: benchmark-results + continue-on-error: true + + - uses: actions/github-script@v9 + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + script: | + const fs = require('fs'); + const marker = ''; + + const body = fs.existsSync('comment.md') + ? fs.readFileSync('comment.md', 'utf8') + : [marker, '## Benchmark Report', + `:warning: The benchmarks did not run to completion, see the [workflow run](${process.env.RUN_URL}).`].join('\n'); + + // PR comment needs pull-requests:write, downgraded to read-only on + // fork PRs — don't fail the job when the write errors out. + try { + const { data: comments } = await github.rest.issues.listComments({ + ...context.repo, issue_number: context.issue.number, per_page: 100, + }); + const existing = comments.find(c => c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); + } + } catch (e) { + core.warning(`Could not post PR comment (likely a fork PR with a read-only token): ${e.message}`); + } + + # Separate from the job that runs the code, which keeps a read-only token. + publish: + if: ${{ github.event_name == 'push' }} + needs: benchmark + runs-on: ubuntu-latest + permissions: + contents: write + steps: + # github-action-benchmark commits to gh-pages from the workspace's clone + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/download-artifact@v8 + with: + name: benchmark-results + path: ${{ runner.temp }}/bench + + - name: Publish first-call counts + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: First call + tool: customSmallerIsBetter + output-file-path: ${{ runner.temp }}/bench/first_call.json + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + comment-on-alert: false + + - name: Publish successive-call counts + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: Successive calls + tool: customSmallerIsBetter + output-file-path: ${{ runner.temp }}/bench/successive_call.json + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + comment-on-alert: false + + - name: Publish the counts pull requests compare against + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + git fetch --depth=1 origin gh-pages + git switch --force-create gh-pages FETCH_HEAD + cp "${RUNNER_TEMP}/bench/head.json" benchmarks/baseline.json + git add benchmarks/baseline.json + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -m "Update the benchmark baseline to ${GITHUB_SHA}" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:gh-pages diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ac99203e..247bf222 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: - name: Check C++ formatting run: | sudo apt-get install -y clang-format-18 - find src test/cpp -name '*.cxx' -o -name '*.h' \ + find src test/cpp benchmarks/cpp -name '*.cxx' -o -name '*.h' \ | xargs clang-format-18 --dry-run -Werror ruff: diff --git a/benchmarks/Makefile b/benchmarks/Makefile new file mode 100644 index 00000000..114b5779 --- /dev/null +++ b/benchmarks/Makefile @@ -0,0 +1,30 @@ +# the C++ side of the benchmarks builds into a shared library next to its +# sources (load_reflection_info resolves the header as a sibling of the +# loaded Dict library) +soext = .so +ifeq ($(OS),Windows_NT) + soext = .dll +endif + +dictnames = calls + +dicts = $(addprefix cpp/,$(addsuffix Dict$(soext),$(dictnames))) + +all : $(dicts) + +# CPPFLAGS can point at the valgrind headers, for counting instructions +cppflags= -std=c++17 -O2 -fPIC $(CPPFLAGS) + +PLATFORM := $(shell uname -s) +ifeq ($(PLATFORM),Darwin) + cppflags+=-dynamiclib -single_module -undefined dynamic_lookup +endif + +# a worker can load the library while another rebuilds it, so publish it whole +cpp/%Dict$(soext): cpp/%.cxx cpp/%.h cpp/%_api.h cpp/%_api_impl.h + $(CXX) $(cppflags) -shared -o $@.$$$$.tmp $< && mv -f $@.$$$$.tmp $@ + +.PHONY: clean + +clean: + -rm -f $(dicts) cpp/*.tmp diff --git a/benchmarks/callgrind.sh b/benchmarks/callgrind.sh new file mode 100755 index 00000000..5220bebe --- /dev/null +++ b/benchmarks/callgrind.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Count the instructions of every benchmark in test_calls.py under callgrind +# and write them to a JSON file for report.py. +# +# ./callgrind.sh OUT.json [report.py collect options] [-- pytest args] +# +# Needs valgrind on PATH and setarch; CPPFLAGS (for make) may point at the +# valgrind headers. PYTHON picks the interpreter (default: python on PATH). +# In docker, ASLR can only be turned off with +# --security-opt seccomp=unconfined. +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +out="$(realpath -m "$1")" +shift +collect_args=() +while [[ $# -gt 0 && "$1" != "--" ]]; do + collect_args+=("$1") + shift +done +[[ $# -gt 0 ]] && shift +python="$(command -v "${PYTHON:-python}")" + +dumps="$(mktemp -d)" +trap 'rm -rf "${dumps}"' EXIT + +cd "${here}" +# always: make cannot tell that a library built without CPPFLAGS (say, by a +# plain pytest run) lacks the markers +make -B cpp/callsDict.so +# Counts depend on memory layout (hash tables keyed by address), so run with +# ASLR off and a fixed environment: its size moves the stack. A fixed hash +# seed keeps dict and set layouts the same too. +valgrind="$(command -v valgrind)" +run_env=( + PATH=/usr/local/bin:/usr/bin:/bin + HOME=/tmp + PYTHONHASHSEED=0 + CPPJIT_BENCH_CALLGRIND=1 +) +if [[ -n "${CPPINTEROP_EXTRA_INTERPRETER_ARGS:-}" ]]; then + run_env+=(CPPINTEROP_EXTRA_INTERPRETER_ARGS="${CPPINTEROP_EXTRA_INTERPRETER_ARGS}") +fi +env -i "${run_env[@]}" setarch -R "${valgrind}" -q --tool=callgrind \ + --collect-atstart=no \ + --callgrind-out-file="${dumps}/callgrind.out" \ + "${python}" -m pytest test_calls.py -p no:cacheprovider -q "$@" +"${python}" report.py collect "${dumps}" -o "${out}" \ + ${collect_args[@]+"${collect_args[@]}"} diff --git a/benchmarks/cpp/calls.cxx b/benchmarks/cpp/calls.cxx new file mode 100644 index 00000000..ca83571e --- /dev/null +++ b/benchmarks/cpp/calls.cxx @@ -0,0 +1,29 @@ +#include "calls.h" + +// Markers for counting instructions under callgrind (see ../callgrind.sh), +// called through ctypes so that cppjit's own call path stays out of them. +// Without the valgrind headers they do nothing, and bench_on_valgrind() +// says so. +#if __has_include() +#include + +extern "C" int bench_on_valgrind() { return RUNNING_ON_VALGRIND ? 1 : 0; } +extern "C" void bench_toggle_collect() { CALLGRIND_TOGGLE_COLLECT; } +extern "C" void bench_dump(const char* name) { CALLGRIND_DUMP_STATS_AT(name); } +#else +extern "C" int bench_on_valgrind() { return -1; } +extern "C" void bench_toggle_collect() {} +extern "C" void bench_dump(const char*) {} +#endif + +#define BENCH_NS hot +#include "calls_api_impl.h" +#undef BENCH_NS + +#define BENCH_NS warmup +#include "calls_api_impl.h" +#undef BENCH_NS + +#define BENCH_NS cold +#include "calls_api_impl.h" +#undef BENCH_NS diff --git a/benchmarks/cpp/calls.h b/benchmarks/cpp/calls.h new file mode 100644 index 00000000..cacebc65 --- /dev/null +++ b/benchmarks/cpp/calls.h @@ -0,0 +1,63 @@ +#ifndef CPPJIT_BENCHMARKS_CALLS_H +#define CPPJIT_BENCHMARKS_CALLS_H + +#include +#include +#include + +// Argument types shared by every copy of the API, so that the benchmarks can +// build their arguments once, outside the timed region. +struct Point { + double x = 0., y = 0.; +}; + +class Widget { +public: + Widget() = default; + int id = 0; +}; + +struct Particle { + Point position, velocity; + double mass = 1.; +}; + +enum Color { Red, Green, Blue }; +enum class Shape : int { Circle, Square, Triangle }; + +// (tag, type) of every primitive type the benchmarks pass across +#define BENCH_PRIMITIVES(X) \ + X(bool, bool) \ + X(char, char) \ + X(schar, signed char) \ + X(uchar, unsigned char) \ + X(wchar, wchar_t) \ + X(char16, char16_t) \ + X(char32, char32_t) \ + X(short, short) \ + X(ushort, unsigned short) \ + X(int, int) \ + X(uint, unsigned int) \ + X(long, long) \ + X(ulong, unsigned long) \ + X(llong, long long) \ + X(ullong, unsigned long long) \ + X(float, float) \ + X(double, double) \ + X(ldouble, long double) + +// The copies of the API: `hot` serves the successive-call benchmarks, and the +// first-call benchmarks call `warmup` before they measure `cold`. +#define BENCH_NS hot +#include "calls_api.h" +#undef BENCH_NS + +#define BENCH_NS warmup +#include "calls_api.h" +#undef BENCH_NS + +#define BENCH_NS cold +#include "calls_api.h" +#undef BENCH_NS + +#endif // CPPJIT_BENCHMARKS_CALLS_H diff --git a/benchmarks/cpp/calls_api.h b/benchmarks/cpp/calls_api.h new file mode 100644 index 00000000..54b5e126 --- /dev/null +++ b/benchmarks/cpp/calls_api.h @@ -0,0 +1,64 @@ +// The benchmarked API. calls.h includes this once per copy, each time with a +// different BENCH_NS, so that the first-call benchmarks find functions that +// were never called before. No include guard on purpose. +// +// The bodies only hand their input back: the benchmarks measure cppjit's +// call path, not the work the C++ function does. + +namespace BENCH_NS { + +// Single, non-templated overload: one per primitive type, taken by value, +// pointer, reference and const reference, always returning the value. +#define BENCH_DECLARE_PASS(tag, T) \ + T pass_##tag(T x); \ + T pass_ptr_##tag(T* x); \ + T pass_ref_##tag(T& x); \ + T pass_cref_##tag(const T& x); +BENCH_PRIMITIVES(BENCH_DECLARE_PASS) +#undef BENCH_DECLARE_PASS + +std::string pass_string(std::string x); +std::string pass_string_cref(const std::string& x); +std::string_view pass_string_view(std::string_view x); +const char* pass_cstring(const char* x); + +// Single, non-templated overload taking several class and enum arguments. +Point pass_point(Point p); +Color pass_color(Color c); +Shape pass_shape(Shape s); +int pass_value_args(Point p, Color c, Shape s); +int pass_ref_args(const Point& p, Widget& w, Shape s); +int pass_ptr_args(Point* p, const Widget* w, Color c); +int pass_mixed_args(const Particle& pa, Widget* w, Point& p, Color c, Shape s); + +// One name, several non-templated overloads over classes and enums, mixing +// references and pointers. Each overload returns a different value, so the +// benchmarks can check that resolution picked the intended one. +int overloaded(const Point& p, Color c); +int overloaded(Widget* w, Shape s); +int overloaded(Particle& pa, const Point* p, Color c); +int overloaded(const Widget& w, const Particle& pa, Point* p); +int overloaded(Color c, Shape s); + +// Single, templated overload, instantiated with primitive types: deduced +// from the arguments, and given explicitly. Two templates, so that an +// explicit case never finds its instantiation made by a deduced one. +template T tmpl_deduce(T x) { return x; } +template T tmpl_pass(T x) { return x; } + +// Several templated overloads, instantiated with classes and enums. +template int tmpl_overloaded(const T&) { return 1; } +template int tmpl_overloaded(T*, Color) { return 2; } +template int tmpl_overloaded(const T&, U&, Shape) { + return 3; +} + +// Higher order functions called with a Python callable, overloaded and +// mixing templated and non-templated overloads. Each calls the callback once. +int invoke(int (*cb)(int), int x); +double invoke(double (*cb)(double), double x); +int invoke(int (*cb)(const Point&), const Point& p); +int invoke(const std::function& cb, Widget& w, Color c); +template T invoke(T (*cb)(T, T), T x, T y) { return cb(x, y); } + +} // namespace BENCH_NS diff --git a/benchmarks/cpp/calls_api_impl.h b/benchmarks/cpp/calls_api_impl.h new file mode 100644 index 00000000..46a4c2a6 --- /dev/null +++ b/benchmarks/cpp/calls_api_impl.h @@ -0,0 +1,42 @@ +// Definitions for calls_api.h, included by calls.cxx once per BENCH_NS copy. +// No include guard on purpose. + +namespace BENCH_NS { + +#define BENCH_DEFINE_PASS(tag, T) \ + T pass_##tag(T x) { return x; } \ + T pass_ptr_##tag(T* x) { return *x; } \ + T pass_ref_##tag(T& x) { return x; } \ + T pass_cref_##tag(const T& x) { return x; } +BENCH_PRIMITIVES(BENCH_DEFINE_PASS) +#undef BENCH_DEFINE_PASS + +std::string pass_string(std::string x) { return x; } +std::string pass_string_cref(const std::string& x) { return x; } +std::string_view pass_string_view(std::string_view x) { return x; } +const char* pass_cstring(const char* x) { return x; } + +Point pass_point(Point p) { return p; } +Color pass_color(Color c) { return c; } +Shape pass_shape(Shape s) { return s; } +int pass_value_args(Point, Color, Shape) { return 0; } +int pass_ref_args(const Point&, Widget&, Shape) { return 0; } +int pass_ptr_args(Point*, const Widget*, Color) { return 0; } +int pass_mixed_args(const Particle&, Widget*, Point&, Color, Shape) { + return 0; +} + +int overloaded(const Point&, Color) { return 1; } +int overloaded(Widget*, Shape) { return 2; } +int overloaded(Particle&, const Point*, Color) { return 3; } +int overloaded(const Widget&, const Particle&, Point*) { return 4; } +int overloaded(Color, Shape) { return 5; } + +int invoke(int (*cb)(int), int x) { return cb(x); } +double invoke(double (*cb)(double), double x) { return cb(x); } +int invoke(int (*cb)(const Point&), const Point& p) { return cb(p); } +int invoke(const std::function& cb, Widget& w, Color c) { + return cb(w, c); +} + +} // namespace BENCH_NS diff --git a/benchmarks/report.py b/benchmarks/report.py new file mode 100644 index 00000000..90c0510b --- /dev/null +++ b/benchmarks/report.py @@ -0,0 +1,266 @@ +"""Turn the benchmarks' callgrind counts into results and reports. + +collect: read the dumps of a callgrind.sh run into a results JSON: + {"benchmarks": {test name: instructions}, "commit": ..., + "fingerprint": ...}. +compare: compare head's results against base's, write a markdown report + (the PR comment) and exit 1 when a benchmark got slower than the + threshold allows. +export: convert results to github-action-benchmark's customSmallerIsBetter + format, one file per kind of call. +""" + +import argparse +import glob +import json +import math +import os +import sys +from collections.abc import Callable +from typing import Any + +# kind, group, case, instructions +Benchmark = tuple[str, str, str, int] +# kind, group, case, base's instructions, head's, head/base +Row = tuple[str, str, str, int, int, float] + +KINDS = ("first_call", "successive_call") +SHORT = {"first_call": "first", "successive_call": "hot"} +TRIGGER = "desc: Trigger: Client Request: " + + +def collect(args: argparse.Namespace) -> int: + counts: dict[str, int] = {} + for path in glob.glob(os.path.join(args.dumps, "callgrind.out*")): + name: str | None = None + total: int | None = None + with open(path) as f: + for line in f: + if line.startswith(TRIGGER): + name = line[len(TRIGGER) :].rstrip("\n") + elif line.startswith(("totals:", "summary:")): + total = int(line.split()[1]) + if name is not None and total is not None: + counts[name] = total + baseline = counts.pop("baseline", None) + if baseline is None or not counts: + sys.exit("%s has no benchmark dumps; did callgrind run?" % args.dumps) + results = { + "commit": args.commit, + "fingerprint": args.fingerprint, + "benchmarks": {k: max(v - baseline, 0) for k, v in sorted(counts.items())}, + } + with open(args.output, "w") as f: + json.dump(results, f, indent=1) + return 0 + + +def load(path: str) -> tuple[dict[str, Any], dict[str, Benchmark]]: + """The results and {name: (kind, group, case, count)} of their benchmarks.""" + + with open(path) as f: + results = json.load(f) + benchmarks: dict[str, Benchmark] = {} + for name, count in results["benchmarks"].items(): + # name is test_[/] + test, _, param = name.partition("[") + group, _, case = param[:-1].partition("/") + benchmarks[name] = (test[len("test_") :], group, case, count) + return results, benchmarks + + +def fmt_count(n: float) -> str: + for scale, suffix in ((1e9, " G"), (1e6, " M"), (1e3, " k")): + if n >= scale: + return "%.4g%s" % (n / scale, suffix) + return "%d" % n + + +def fmt_change(ratio: float) -> str: + return "%+.2f%%" % ((ratio - 1) * 100) + + +def geomean(values: list[int]) -> float: + # a count can be 0 when a call costs no more than the markers + return math.exp(sum(math.log(max(v, 1)) for v in values) / len(values)) + + +def compare(args: argparse.Namespace) -> int: + base_results, base = load(args.base) + head_results, head = load(args.head) + limit = 1 + args.threshold / 100 + rows: list[Row] = [] + for name in sorted(set(base) & set(head)): + kind, group, case, b = base[name] + h = head[name][3] + rows.append((kind, group, case, b, h, max(h, 1) / max(b, 1))) + slower = [r for r in rows if r[5] > limit] + faster = [r for r in rows if r[5] < 1 / limit] + added = sorted(set(head) - set(base)) + removed = sorted(set(base) - set(head)) + + base_label = "`%s`" % args.base_name + head_label = "`%s`" % args.head_name + + out = ["", "## Benchmark Report"] + if slower: + out.append( + ":x: %d of %d benchmarks run more than %g%% more instructions " + "than %s." % (len(slower), len(rows), args.threshold, base_label) + ) + else: + out.append( + ":white_check_mark: No benchmark runs more than %g%% more " + "instructions than %s." % (args.threshold, base_label) + ) + base_sha = base_results.get("commit") + head_sha = head_results.get("commit") + if base_sha and head_sha: + url = "%s/commit/" % args.repo_url if args.repo_url else None + + def commit(sha: str) -> str: + return "[`%s`](%s%s)" % (sha[:7], url, sha) if url else "`%s`" % sha[:7] + + out.append( + "Comparing base (%s) to head (%s), in instructions counted by " + "callgrind." % (commit(base_sha), commit(head_sha)) + ) + if args.note: + out.append(args.note) + if args.charts_url: + out.append("History of `main`: [benchmark charts](%s)." % args.charts_url) + out.append("") + + # codecov-style summary: one line per kind and group with the geometric + # mean of its counts; '-' marks a regression and '+' an improvement + def line(sign: str, label: str, b: str, h: str, change: str) -> str: + return "%s %-26s%11s%11s%10s" % (sign, label, b, h, change) + + width = len(line(" ", "", "", "", "")) + out += [ + "```diff", + "@@%s@@" % " Benchmark Diff (geomean) ".center(width - 4), + "##%25s%11s%11s%8s ##" % ("", args.base_name[:10], args.head_name[:10], "+/-"), + "=" * width, + ] + for kind in KINDS: + for group in sorted({r[1] for r in rows if r[0] == kind}): + sel = [r for r in rows if r[0] == kind and r[1] == group] + base_mean = geomean([r[3] for r in sel]) + head_mean = geomean([r[4] for r in sel]) + ratio = head_mean / base_mean + sign = "-" if ratio > limit else "+" if ratio < 1 / limit else " " + label = "%s/%s" % (SHORT[kind], group) + out.append( + line( + sign, + label, + fmt_count(base_mean), + fmt_count(head_mean), + fmt_change(ratio), + ) + ) + out.append("=" * width) + out.append(line("-", "Slower", "", "", str(len(slower)))) + out.append(line("+", "Faster", "", "", str(len(faster)))) + unchanged = len(rows) - len(slower) - len(faster) + out.append(line(" ", "Unchanged", "", "", str(unchanged))) + if added: + out.append(line(" ", "New", "", "", str(len(added)))) + if removed: + out.append(line(" ", "Removed", "", "", str(len(removed)))) + out += ["```", ""] + + def table(selected: list[Row]) -> list[str]: + lines = [ + "| Benchmark | %s | %s | Δ | |" % (base_label, head_label), + "|---|---:|---:|---:|---|", + ] + for kind, group, case, b, h, ratio in selected: + mark = ":x:" if ratio > limit else ":rocket:" if ratio < 1 / limit else "" + lines.append( + "| %s/%s/`%s` | %s | %s | `%s` | %s |" + % ( + SHORT[kind], + group, + case, + fmt_count(b), + fmt_count(h), + fmt_change(ratio), + mark, + ) + ) + return lines + + changed = sorted(slower, key=lambda r: -r[5]) + sorted(faster, key=lambda r: r[5]) + if changed: + out += table(changed) + [""] + out += ["
All benchmarks", ""] + out += table(rows) + for name in added: + out.append("| `%s` | | %s | new | |" % (name, fmt_count(head[name][3]))) + for name in removed: + out.append("| `%s` | %s | | removed | |" % (name, fmt_count(base[name][3]))) + out += ["", "
", ""] + + report = "\n".join(out) + if args.markdown: + with open(args.markdown, "w") as f: + f.write(report) + else: + print(report) + return 1 if slower else 0 + + +def export(args: argparse.Namespace) -> int: + _, benchmarks = load(args.results) + for kind in KINDS: + data = [ + {"name": "%s/%s" % (group, case), "unit": "instructions", "value": n} + for k, group, case, n in sorted(benchmarks.values()) + if k == kind + ] + with open(getattr(args, kind), "w") as f: + json.dump(data, f, indent=1) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("collect", help="callgrind dumps to results") + p.add_argument("dumps", help="directory of callgrind.out* files") + p.add_argument("-o", "--output", required=True) + p.add_argument("--commit", help="the measured commit, for the report") + p.add_argument( + "--fingerprint", + help="the toolchain and environment; counts compare only within one", + ) + p.set_defaults(func=collect) + + p = sub.add_parser("compare", help="compare head against base") + p.add_argument("--base", required=True, metavar="JSON") + p.add_argument("--head", required=True, metavar="JSON") + p.add_argument("--threshold", type=float, default=5.0, help="percent") + p.add_argument("--base-name", default="main") + p.add_argument("--head-name", default="head") + p.add_argument("--repo-url", help="for commit links") + p.add_argument("--charts-url") + p.add_argument("--note", help="a line to add under the summary") + p.add_argument("--markdown", help="write the report here, not to stdout") + p.set_defaults(func=compare) + + p = sub.add_parser("export", help="github-action-benchmark input") + p.add_argument("results", metavar="JSON") + for kind in KINDS: + p.add_argument("--" + kind.replace("_", "-"), dest=kind, required=True) + p.set_defaults(func=export) + + args = parser.parse_args(argv) + func: Callable[[argparse.Namespace], int] = args.func + return func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/test_calls.py b/benchmarks/test_calls.py new file mode 100644 index 00000000..aaf9a6b1 --- /dev/null +++ b/benchmarks/test_calls.py @@ -0,0 +1,415 @@ +"""Benchmarks of calling C++ functions from Python, in instructions. + +Every case is measured twice: + +- test_first_call: the first call of a function, which also pays for + cppjit's lazy setup (wrapper generation, converter lookup, template + instantiation, overload resolution). It calls the function from the `cold` + copy of the API, after calling it once from the `warmup` copy: that call + absorbs the process-wide first-use costs (say, the first converter for a + type), which would otherwise land on whichever case happens to run first. +- test_successive_call: a call of the `hot` copy's function after a first one. +""" + +import ctypes +import gc +import os +import subprocess +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import cppjit +import pytest +from cppjit import gbl + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# build the C++ side on demand +subprocess.run(["make", "cpp/callsDict.so"], cwd=HERE, check=True) +cppjit.load_reflection_info(os.path.join(HERE, "cpp", "callsDict")) + +# get and check of callgrind marker functions +_markers: ctypes.CDLL = ctypes.CDLL(os.path.join(HERE, "cpp", "callsDict.so")) +_markers.bench_dump.argtypes = [ctypes.c_char_p] +_toggle = _markers.bench_toggle_collect + +if ( + os.environ.get("CPPJIT_BENCH_CALLGRIND") == "1" + and _markers.bench_on_valgrind() != 1 +): + raise RuntimeError( + "counting instructions needs callgrind.sh and a callsDict.so built with " + "the valgrind headers (make CPPFLAGS=-I)" + ) + +Invoker = Callable[[Any, tuple[Any, ...]], Any] +Args = Callable[[], tuple[Any, ...]] + + +def _measure(name: str, invoke: Invoker, fn: Any, args: tuple[Any, ...]) -> Any: + gc.collect() + gc.disable() + _toggle() + result = invoke(fn, args) + _toggle() + gc.enable() + _markers.bench_dump(name.encode()) + return result + + +def _noop(fn: Any, args: tuple[Any, ...]) -> None: + pass + + +# The markers' own cost, which collect subtracts from every count. The empty +# measurement mirrors _measure, so that only the call itself remains. +_measure("baseline", _noop, None, ()) + +# keeping arguments stay alive: +# a callback that dies hands its wrapper to the next one +_keepalive: list[tuple[Any, ...]] = [] + +_UNCHECKED: Any = object() + + +@dataclass(frozen=True) +class Case: + group: str + func: str + args: Args + expect: Any = _UNCHECKED + # explicit template arguments, as in func[tmpl](*args) + tmpl: str | None = None + name: str | None = None + + @property + def id(self) -> str: + return "%s/%s" % (self.group, self.name or self.func) + + def invoker(self) -> Invoker: + if self.tmpl is None: + return lambda fn, args: fn(*args) + tmpl = self.tmpl + return lambda fn, args: fn[tmpl](*args) + + +def primitive_cases() -> list[Case]: + # tag: (result, value arguments, pointer arguments, reference arguments) + # The value arguments also serve the const reference. The char types' + # pointers are C strings, so they take a str; the other pointers and all + # references take a ctypes object. + primitives: dict[str, tuple[Any, Args, Args, Args]] = { + "bool": ( + True, + lambda: (True,), + lambda: (ctypes.c_bool(True),), + lambda: (ctypes.c_bool(True),), + ), + "char": ("a", lambda: ("a",), lambda: ("a",), lambda: (ctypes.c_char(b"a"),)), + "schar": ("\x01", lambda: (1,), lambda: ("\x01",), lambda: (ctypes.c_byte(1),)), + "uchar": ( + "\x01", + lambda: (1,), + lambda: (ctypes.c_ubyte(1),), + lambda: (ctypes.c_ubyte(1),), + ), + "wchar": ("a", lambda: ("a",), lambda: ("a",), lambda: (ctypes.c_wchar("a"),)), + "char16": ( + "a", + lambda: ("a",), + lambda: ("a",), + lambda: (ctypes.c_uint16(ord("a")),), + ), + "char32": ( + "a", + lambda: ("a",), + lambda: ("a",), + lambda: (ctypes.c_uint32(ord("a")),), + ), + "short": ( + 1, + lambda: (1,), + lambda: (ctypes.c_short(1),), + lambda: (ctypes.c_short(1),), + ), + "ushort": ( + 1, + lambda: (1,), + lambda: (ctypes.c_ushort(1),), + lambda: (ctypes.c_ushort(1),), + ), + "int": ( + 1, + lambda: (1,), + lambda: (ctypes.c_int(1),), + lambda: (ctypes.c_int(1),), + ), + "uint": ( + 1, + lambda: (1,), + lambda: (ctypes.c_uint(1),), + lambda: (ctypes.c_uint(1),), + ), + "long": ( + 1, + lambda: (1,), + lambda: (ctypes.c_long(1),), + lambda: (ctypes.c_long(1),), + ), + "ulong": ( + 1, + lambda: (1,), + lambda: (ctypes.c_ulong(1),), + lambda: (ctypes.c_ulong(1),), + ), + "llong": ( + 1, + lambda: (1,), + lambda: (ctypes.c_longlong(1),), + lambda: (ctypes.c_longlong(1),), + ), + "ullong": ( + 1, + lambda: (1,), + lambda: (ctypes.c_ulonglong(1),), + lambda: (ctypes.c_ulonglong(1),), + ), + "float": ( + 1.0, + lambda: (1.0,), + lambda: (ctypes.c_float(1.0),), + lambda: (ctypes.c_float(1.0),), + ), + "double": ( + 1.0, + lambda: (1.0,), + lambda: (ctypes.c_double(1.0),), + lambda: (ctypes.c_double(1.0),), + ), + "ldouble": ( + 1.0, + lambda: (1.0,), + lambda: (ctypes.c_longdouble(1.0),), + lambda: (ctypes.c_longdouble(1.0),), + ), + } + # cppjit has no converter for const references to the wide char types + no_cref = {"wchar", "char16", "char32"} + + cases: list[Case] = [] + for tag, (result, args, ptr_args, ref_args) in primitives.items(): + cases.append(Case("primitive", "pass_" + tag, args, result)) + cases.append(Case("primitive", "pass_ptr_" + tag, ptr_args, result)) + cases.append(Case("primitive", "pass_ref_" + tag, ref_args, result)) + if tag not in no_cref: + cases.append(Case("primitive", "pass_cref_" + tag, args, result)) + return cases + + +def string_cases() -> list[Case]: + return [ + Case("string", "pass_string", lambda: ("abc",), "abc"), + Case("string", "pass_string_cref", lambda: ("abc",), "abc"), + Case("string", "pass_string_view", lambda: ("abc",), "abc"), + Case("string", "pass_cstring", lambda: ("abc",), "abc"), + ] + + +def class_cases() -> list[Case]: + return [ + Case("class", "pass_point", lambda: (gbl.Point(),)), + Case("class", "pass_color", lambda: (gbl.Blue,), gbl.Blue), + Case("class", "pass_shape", lambda: (gbl.Shape.Square,), gbl.Shape.Square), + Case( + "class", + "pass_value_args", + lambda: (gbl.Point(), gbl.Red, gbl.Shape.Circle), + 0, + ), + Case( + "class", + "pass_ref_args", + lambda: (gbl.Point(), gbl.Widget(), gbl.Shape.Circle), + 0, + ), + Case("class", "pass_ptr_args", lambda: (gbl.Point(), gbl.Widget(), gbl.Red), 0), + Case( + "class", + "pass_mixed_args", + lambda: ( + gbl.Particle(), + gbl.Widget(), + gbl.Point(), + gbl.Red, + gbl.Shape.Circle, + ), + 0, + ), + ] + + +def overload_cases() -> list[Case]: + args: list[Args] = [ + lambda: (gbl.Point(), gbl.Red), + lambda: (gbl.Widget(), gbl.Shape.Circle), + lambda: (gbl.Particle(), gbl.Point(), gbl.Red), + lambda: (gbl.Widget(), gbl.Particle(), gbl.Point()), + lambda: (gbl.Red, gbl.Shape.Circle), + ] + return [ + Case("overload", "overloaded", a, i + 1, name="overloaded_%d" % (i + 1)) + for i, a in enumerate(args) + ] + + +def template_cases() -> list[Case]: + cases = [ + Case("template", "tmpl_deduce", lambda: (1,), 1, name="tmpl_deduce(int)"), + Case("template", "tmpl_deduce", lambda: (1.0,), 1.0, name="tmpl_deduce(float)"), + Case( + "template", "tmpl_deduce", lambda: (True,), True, name="tmpl_deduce(bool)" + ), + ] + explicit: list[tuple[str, Args, Any]] = [ + ("bool", lambda: (True,), True), + ("char", lambda: ("a",), "a"), + ("short", lambda: (1,), 1), + ("unsigned int", lambda: (1,), 1), + ("long", lambda: (1,), 1), + ("unsigned long long", lambda: (1,), 1), + ("float", lambda: (1.0,), 1.0), + ("double", lambda: (1.0,), 1.0), + ("long double", lambda: (1.0,), 1.0), + ] + for tmpl, args, result in explicit: + cases.append( + Case( + "template", + "tmpl_pass", + args, + result, + tmpl=tmpl, + name="tmpl_pass[%s]" % tmpl, + ) + ) + return cases + + +def template_overload_cases() -> list[Case]: + # (case name, arguments, overload) + calls: list[tuple[str, Args, int]] = [ + ("tmpl_overloaded_1(Point)", lambda: (gbl.Point(),), 1), + ("tmpl_overloaded_2(Point)", lambda: (gbl.Point(), gbl.Red), 2), + ("tmpl_overloaded_1(Widget)", lambda: (gbl.Widget(),), 1), + ("tmpl_overloaded_2(Widget)", lambda: (gbl.Widget(), gbl.Red), 2), + ("tmpl_overloaded_1(Particle)", lambda: (gbl.Particle(),), 1), + ("tmpl_overloaded_2(Particle)", lambda: (gbl.Particle(), gbl.Red), 2), + ( + "tmpl_overloaded_3(Point,Widget)", + lambda: (gbl.Point(), gbl.Widget(), gbl.Shape.Circle), + 3, + ), + ( + "tmpl_overloaded_3(Particle,Point)", + lambda: (gbl.Particle(), gbl.Point(), gbl.Shape.Circle), + 3, + ), + ] + return [ + Case("template_overload", "tmpl_overloaded", args, overload, name=name) + for name, args, overload in calls + ] + + +def callback_cases() -> list[Case]: + # The args build a new Python callable per call, so that the first call + # also pays for wrapping it; the hot benchmark reuses one callable. + return [ + Case( + "callback", + "invoke", + lambda: (lambda x: x, 1), + 1, + name="invoke(int(*)(int))", + ), + Case( + "callback", + "invoke", + lambda: (lambda x: x, 1.0), + 1.0, + name="invoke(double(*)(double))", + ), + Case( + "callback", + "invoke", + lambda: (lambda p: 3, gbl.Point()), + 3, + name="invoke(int(*)(const Point&))", + ), + Case( + "callback", + "invoke", + lambda: (lambda w, c: 4, gbl.Widget(), gbl.Red), + 4, + name="invoke(std::function)", + ), + Case( + "callback", + "invoke", + lambda: (lambda x, y: x, 5, 6), + 5, + tmpl="int", + name="invoke[int](T(*)(T,T))", + ), + Case( + "callback", + "invoke", + lambda: (lambda x, y: x, 5.0, 6.0), + 5.0, + tmpl="double", + name="invoke[double](T(*)(T,T))", + ), + ] + + +def _all_cases() -> list[Case]: + return ( + primitive_cases() + + string_cases() + + class_cases() + + overload_cases() + + template_cases() + + template_overload_cases() + + callback_cases() + ) + + +CASES = _all_cases() +cases = pytest.mark.parametrize("case", CASES, ids=[c.id for c in CASES]) + + +def _check(case: Case, result: Any) -> None: + if case.expect is not _UNCHECKED: + assert result == case.expect, case.id + + +def _call(case: Case, ns: Any) -> tuple[Any, tuple[Any, ...]]: + fn, args = getattr(ns, case.func), case.args() + _keepalive.append(args) + return fn, args + + +@cases +def test_first_call(request: pytest.FixtureRequest, case: Case) -> None: + invoke = case.invoker() + _check(case, invoke(*_call(case, cppjit.gbl.warmup))) + _check(case, _measure(request.node.name, invoke, *_call(case, cppjit.gbl.cold))) + + +@cases +def test_successive_call(request: pytest.FixtureRequest, case: Case) -> None: + invoke = case.invoker() + fn, args = _call(case, cppjit.gbl.hot) + _check(case, invoke(fn, args)) + _check(case, _measure(request.node.name, invoke, fn, args)) diff --git a/changelog/2026-10-21-Benchmarks.md b/changelog/2026-10-21-Benchmarks.md new file mode 100644 index 00000000..dfe6b929 --- /dev/null +++ b/changelog/2026-10-21-Benchmarks.md @@ -0,0 +1,86 @@ +# 2026-10-21: Benchmarks of calls into C++ + +cppjit now has a benchmark suite for what calling C++ functions from Python +costs, and CI that catches pull requests that make those calls slower. It +lets work such as the new overload resolution +([#150](https://github.com/compiler-research/cppjit/pull/150)) and memory +management ([#114](https://github.com/compiler-research/cppjit/pull/114)) +show its effect on call overhead. + +## What is measured + +`benchmarks/test_calls.py` counts the instructions that +[callgrind](https://valgrind.org/docs/manual/cl-manual.html) sees for a +single call. Instruction counts do not depend on the machine's load, so one +call per case is enough, and runs on different machines with the same +toolchain can be compared. + +Every case is measured twice: + +- `test_first_call`: the first call of a function, including cppjit's lazy + setup (wrapper generation, template instantiation, overload resolution). + It calls a never-called copy of the function (namespace `cold`), after + warming up the process on another copy (`warmup`). +- `test_successive_call`: a call of an already-called function (`hot`). + +The cases cover: + +- every primitive type, passed by value, pointer, reference and const + reference; +- strings (`std::string`, `std::string_view`, `const char*`); +- classes and enums, as several arguments mixing values, references and + pointers; +- a non-templated overload set over classes and enums; +- a template instantiated with primitive types, deduced and explicit; +- overloaded templates instantiated with classes; +- overloaded higher order functions, templated and not, taking a Python + callback. + +The C++ functions only return their input, so the counts are cppjit's +overhead. They live in `benchmarks/cpp/`, and `make` builds them into +`benchmarks/cpp/callsDict.so` on the first run. Every result is checked, so +a case that resolves to the wrong overload fails instead of measuring the +wrong call. + +## Running the benchmarks + +```bash +cd benchmarks +./callgrind.sh counts.json # count everything (~2 min) +./callgrind.sh counts.json -- -k first # only some cases +python -m pytest test_calls.py # no counts, only check results +python report.py compare --base main.json --head counts.json +``` + +`callgrind.sh` needs valgrind and its headers (pass `CPPFLAGS=-I` if +they are not on the default include path). To make counts repeatable it +runs Python with ASLR off and a fixed environment. In docker, turning ASLR +off needs `--security-opt seccomp=unconfined`. + +## CI + +The new `.github/workflows/benchmark.yml` counts on Linux x86_64 with +Release builds, against the CppInterOp commit pinned in +`CPPINTEROP_GIT_TAG`, like the test workflow. + +- A push to `main` publishes its counts to the `gh-pages` branch: charts of + every benchmark over time (with + [github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark), + under `/benchmarks/`) and `benchmarks/baseline.json`. +- A pull request counts only itself and compares against that baseline. + `main` is counted again on the same runner when the baseline is missing, + was counted with another toolchain (the `fingerprint`), or shows a + regression, which this confirms. The job fails when a benchmark runs more + than 5% more instructions than `main`, and the comparison is posted as a + PR comment. + +The charts need a `gh-pages` branch with GitHub Pages serving it. + +## Known limitations + +- Counts are repeatable for one build, but successive calls are sensitive + to the process's memory layout: a change outside the measured call can + move them by a few percent, enough to cross the 5% threshold. +- cppjit has no converter for `const wchar_t&`, `const char16_t&` and + `const char32_t&`, so those cases are left out. A pointer to a char type + is passed as a C string.