diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26225103..2a43ee75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,57 @@ jobs: cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp" cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp" + # The experimental debugger support is off in the other cells, which skip the + # gdb tests of test_debugging.py. + debugger: + name: debugger/ubuntu-24.04/llvm22/py3.14 + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.14' + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends cmake ninja-build g++ git libedit-dev + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + # The DAP tests need gdb 16; Ubuntu 24.04 has gdb 15. + - uses: mamba-org/setup-micromamba@v3 + with: + environment-name: gdb-env + create-args: gdb=16.3 + condarc: | + channels: + - conda-forge + - name: Add gdb to PATH + run: | + # Only gdb: the environment's python must not shadow setup-python's. + mkdir -p "$RUNNER_TEMP/gdb-bin" + ln -s "${MAMBA_ROOT_PREFIX:-$HOME/micromamba}/envs/gdb-env/bin/gdb" "$RUNNER_TEMP/gdb-bin/gdb" + echo "$RUNNER_TEMP/gdb-bin" >> "$GITHUB_PATH" + - uses: compiler-research/ci-workflows/actions/setup-llvm@main + with: + version: '22' + os: ubuntu-24.04 + - name: Build and install CppJIT with the debugger support + run: | + CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" python -m pip install . -v \ + --config-settings=cmake.define.LLVM_DIR="$LLVM_DIR" \ + --config-settings=cmake.define.CPPJIT_ENABLE_DEBUGGER_SUPPORT=ON \ + --config-settings=install.strip=false + - name: Run the debugging tests + working-directory: test + env: + CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++20 + run: | + set -o pipefail + gdb --version | head -1 + python -m pytest -ra --tb=short -p no:cacheprovider test_debugging.py | tee "$RUNNER_TEMP/debugging.log" + # A skip here means gdb or ptrace is not usable, so the cell tests nothing. + ! grep -q "^SKIPPED" "$RUNNER_TEMP/debugging.log" + report: if: ${{ always() && github.event_name == 'pull_request' }} needs: build-test diff --git a/.gitignore b/.gitignore index 65e307ce..de93ab45 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,6 @@ venv/ # Editors / OS .idea/ .vscode/ +# the examples ship their launch configurations +!/examples/*/.vscode/ .DS_Store diff --git a/CMakeLists.txt b/CMakeLists.txt index ac5fe0b7..aeff6e42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,11 @@ include(GNUInstallDirs) # This option won't make a lot of sense since we only ship the shared library in site-packages # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests? option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF) -set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "") -set(CPPINTEROP_GIT_TAG "f4adaa6b50542dd564089fe584d407bf15351c97" CACHE STRING "") +# Experimental: debugging of JIT-compiled C++ with gdb (cppjit.debugger, the +# gdb extension and the libcppjit hooks). Off, the build has none of it. +option(CPPJIT_ENABLE_DEBUGGER_SUPPORT "build the experimental debugger support" OFF) +set(CPPINTEROP_GIT_REPOSITORY "https://github.com/chicagotrading/CppInterOp.git" CACHE STRING "") +set(CPPINTEROP_GIT_TAG "upstreaming/jit-debugging" CACHE STRING "") set(CPPINTEROP_SOURCE_DIR "" CACHE PATH "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source") @@ -77,6 +80,7 @@ target_compile_definitions(cppjit PRIVATE CPPJIT_CLANG_MAJOR="${CPPJIT_INTEROP_CLANG_MAJOR}" CPPJIT_CLANG_INCLUDE_DIR="${CPPJIT_INTEROP_CLANG_DIR}" $<$:CPPJIT_USE_CLING> + $<$:CPPJIT_WITH_DEBUGGER_SUPPORT> ) target_include_directories(cppjit PRIVATE @@ -116,10 +120,22 @@ if(NOT SKBUILD) set_target_properties(cppjit PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/cppjit" ) + set(_cppjit_debugger_tree) + if(CPPJIT_ENABLE_DEBUGGER_SUPPORT) + set(_cppjit_debugger_tree + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_CURRENT_SOURCE_DIR}/python/cppjit-debugger" + "${CMAKE_BINARY_DIR}/python/cppjit/debugger" + COMMAND ${CMAKE_COMMAND} -E rename + "${CMAKE_BINARY_DIR}/python/cppjit/debugger/libcppjit.so-gdb.py" + "${CMAKE_BINARY_DIR}/python/cppjit/libcppjit.so-gdb.py" + ) + endif() add_custom_target(cppjit-python-tree ALL COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}/python/cppjit" "${CMAKE_BINARY_DIR}/python/cppjit" + ${_cppjit_debugger_tree} ) endif() @@ -137,6 +153,19 @@ if(NOT SKBUILD) ) endif() +# The experimental debugger support installs as cppjit.debugger, with the +# gdb auto-load script beside the extension. +if(CPPJIT_ENABLE_DEBUGGER_SUPPORT) + install(DIRECTORY python/cppjit-debugger/ + DESTINATION cppjit/debugger + PATTERN "__pycache__" EXCLUDE + PATTERN "libcppjit.so-gdb.py" EXCLUDE + ) + install(FILES python/cppjit-debugger/libcppjit.so-gdb.py + DESTINATION cppjit + ) +endif() + # An external CppInterOp is consumed in place and nothing of it ships; # the bundled one installs with the builtin headers of the build clang, # laid out as a headers-only resource dir (only include/ ships). diff --git a/README.md b/README.md index 26bb9ba7..7cd6240f 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,29 @@ reflection and JIT compilation API for cppjit. - Domain-specific languages: user-defined "pythonizations" adapt the bindings into Pythonic libraries. +### Debugging JIT-compiled C++ (experimental) + +Build cppjit with `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON` to debug the +JIT-compiled C++ at the source level with gdb, also from Python. The +default build does not have this support. In a build with it, set +`CPPJIT_DEBUGGER_SUPPORT=1` before `import cppjit`. cppjit then compiles +with `-g -O0` and writes each C++ input to a file in +`$TMPDIR/cppjit-debug-` (`CPPINTEROP_JIT_SOURCE_DIR` selects another +directory), so that gdb shows the source and binds `file:line` +breakpoints. + +The gdb extension in `cppjit/debugger/gdb` puts the Python frames between +the C++ frames in backtraces, with their variables (CPython 3.13 or +later), and adds the `cppjit` commands, such as `cppjit py-bt`: + +```bash +CPPJIT_DEBUGGER_SUPPORT=1 gdb -iex "source $(python -c 'import cppjit.debugger as d; print(d.gdb_script())')" \ + -ex 'break -qualified my_function' -ex run --args python my_script.py +``` + +For VS Code, with Python breakpoints and steps between Python and C++, +see [examples/vscode-gdb](examples/vscode-gdb/README.md). + ### Requirements A package manager installation of cppjit (such as pip) requires diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index e83b33ab..d5ffe91d 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -16,7 +16,13 @@ This release supports Python 3.12-3.14 and LLVM 21-22. ## New features -- ... +- Experimental debugger support, in builds with + `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON` only. `CPPJIT_DEBUGGER_SUPPORT=1` + starts a debug session: the JIT compiles with `-g -O0`, and the C++ + inputs go to files that gdb can show. A gdb extension + (`cppjit/debugger/gdb`) shows mixed Python and C++ backtraces with the + variables of both. `examples/vscode-gdb` adds Python breakpoints and + steps between Python and C++ in VS Code, through gdb's DAP server. ## Deprecated features diff --git a/examples/vscode-gdb/.vscode/launch.json b/examples/vscode-gdb/.vscode/launch.json new file mode 100644 index 00000000..e19c839d --- /dev/null +++ b/examples/vscode-gdb/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "cppjit: demo.py (gdb)", + "type": "cppjit-gdb", + "request": "launch", + "program": "python3", + "args": ["${workspaceFolder}/demo.py"], + "cwd": "${workspaceFolder}", + "env": {"CPPJIT_DEBUGGER_SUPPORT": "1"} + }, + { + "name": "cppjit: attach (gdb)", + "type": "cppjit-gdb", + "request": "attach", + "program": "python3", + "pid": "${input:pid}" + } + ], + "inputs": [ + {"id": "pid", "type": "promptString", "description": "Process ID of the Python process"} + ] +} diff --git a/examples/vscode-gdb/README.md b/examples/vscode-gdb/README.md new file mode 100644 index 00000000..55278610 --- /dev/null +++ b/examples/vscode-gdb/README.md @@ -0,0 +1,180 @@ +# Debug JIT-compiled C++ from VS Code + +`demo.py` compiles a C++ function with `cppjit.cppdef`, and a C++ class +from the header `demo.h` with `cppjit.include`, and calls them from +Python. With this example you can stop in the C++ source, step through it, +see its variables, and see the Python lines that called it in the same +call stack. + +The debugger support is experimental. It can change or go away, and it is +not in the default build. + +## Requirements + +- A cppjit built with `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON`. Without it, + cppjit has no `cppjit.debugger` package, no gdb extension and no + debugger hooks. +- Linux, and gdb 16 or later (for its DAP server). Tested with gdb 16.3. +- CPython 3.13 or later for the Python frames in the call stack: the gdb + extension reads CPython's `_Py_DebugOffsets` table, so CPython does not + need debug information. +- A CppInterOp that supports `CPPINTEROP_JIT_SOURCE_DIR`, for breakpoints + by file and line in `cppdef` code. Breakpoints by function name work + without it. +- A ptrace scope that lets gdb debug its child processes + (`/proc/sys/kernel/yama/ptrace_scope` 0 or 1; attach needs 0). + +## Steps + +1. Install the example extension. It has no build step: + + ```sh + ln -s "$PWD/extension" ~/.vscode/extensions/cppjit-gdb + ``` + + Then reload VS Code. + +2. Open this directory in VS Code. In `.vscode/launch.json`, set `program` + to a Python interpreter that can import cppjit. + +3. Set a breakpoint. Click in the gutter of `demo.py`, for example at the + `stats = ...` line in `report`. Or, in Run and Debug, add a function breakpoint + `summarize` (Breakpoints, `+`). Or click in the gutter of `demo.h`, for + example in `Histogram::add`. Or run the demo once, then open the + C++ input from the source directory and click in the gutter: + + ```sh + grep -l summarize "${TMPDIR:-/tmp}/cppjit-debug-$(id -u)"/input_*.cpp + ``` + + The file name is a hash of the code, so the breakpoint stays valid for + the next runs until the code changes. + +4. Start "cppjit: demo.py (gdb)". VS Code stops at the breakpoint. The + call stack shows the C++ frames of `summarize`, then `[py] report` and + `[py] `. Click a `[py]` frame to open `demo.py` at the line + that called the C++ code. + +`CPPJIT_DEBUGGER_SUPPORT=1` in the launch configuration makes cppjit +compile with `-g -O0` and write each C++ input to a file. +`CPPINTEROP_JIT_SOURCE_DIR` selects another directory. Python and C output are line buffered in a +debug session, so the output shows before each stop. + +## Python breakpoints + +gdb cannot bind a breakpoint in a `.py` file, so `extension/dap-proxy.js` +sits between VS Code and gdb. It answers the `setBreakpoints` requests for +Python files itself and writes the lines to a JSON file. The launch request +passes that file to the process in `CPPJIT_PY_BREAKPOINTS`. `cppjit.debugger` +then uses `sys.monitoring` (Python 3.12 or later) to call the native +function `cppjit_debug_break_python` before each of these lines, and the +cppjit gdb extension stops there and shows the Python line as the top +frame. The process reads the file again when it changes, so breakpoints +can be added while it runs. + +The same works without VS Code: write the JSON file +(`{"/path/demo.py": [28]}`), set `CPPJIT_PY_BREAKPOINTS` to its path, and +run under gdb with the extension loaded. + +## Stepping and internal frames + +The call stack hides the internal frames by default: the cppjit and +CppInterOp runtime, CPython, the C runtime, and the `__jc_*` call +wrappers that CppInterOp generates. If a stop is in internal code, the +top frame stays, marked `[internal]`. Set `"internalFrames": "show"` in +the launch configuration to see and step through all frames. + +- At a Python line, Step Over, Step Into and Step Out step through Python + lines. The proxy turns them into `cppjit py-step` and a continue. Step + Over at a `return`, and Step Out, stop in the caller's line. +- Step Into at a Python line that calls C++ stops at the first line of + the C++ function, if it has debug information (JIT code has it). For a + constructor, it stops in the constructor. +- In C++ code, the steps are gdb's own. A step that leaves the outermost + C++ function stops at the Python line that called it, not in the + wrappers. + +In the gdb CLI, `cppjit py-step [into|over|out]` steps through Python +lines and into C++, `cppjit py-step return` goes from C++ code to the +Python line that called it, `cppjit up` and `cppjit down` skip internal frames, and +`set cppjit internal-frames show` shows all frames in `bt`. + +## Breakpoints in headers + +The JIT compiles an inline function, such as `Histogram::add` in +`demo.h`, only when the code first calls it. Until then, gdb finds no code +at a line in it. gdb then moves a breakpoint to the next line with code, +which can be in a different function, for example a constructor. The +cppjit gdb extension disables such a location if the lines between have +code. When the JIT compiles the function, gdb sets the breakpoint again at +the correct line. + +## Python variables + +Each Python frame shows its arguments and its local variables. Module and +class bodies show the names in their namespace, without the `__dunder__` +names. The gdb extension reads the objects from the process memory, so +this also works at a C++ stop and in a core file: + +- `int`, `float`, `bool`, `None`, `str` and `bytes` show like `repr()`. +- Lists, tuples and dicts show a short preview, and expand to their items. +- A C++ object that cppjit returns to Python shows as the C++ object, + as in a C++ frame: it expands to its fields. An object in a smart + pointer, such as `std::shared_ptr`, shows as the `T` that it holds, + or as the smart pointer if it is null. gdb's pretty-printers + apply, for example to `std::vector`, if gdb loads them: gdb loads the + libstdc++ printers only from a directory in its auto-load safe path + (`add-auto-load-safe-path ` in the gdbinit). +- Other objects show ``, and expand to their + instance attributes. + +In a Python frame, the proxy answers a hover, a watch or a console +expression that is a Python name, such as `point.x` or `items[0]`, from +these variables. In the gdb CLI, `bt full` shows them. + +## Why gdb's DAP server + +The extension starts `gdb -i dap` with the cppjit gdb extension loaded. +gdb's own DAP server uses gdb's frame filters for the call stack, so the +Python frames appear in VS Code without more code, and the smoke test +below can drive the same server. The Microsoft C/C++ extension talks to +gdb through MI instead. It can also show JIT code, but frame filters need +`-enable-frame-filters` there, and gdb 16 can hang in +`-stack-list-arguments` when a frame filter meets an optimized-out +artificial `this`. + +`test/test_debugging.py::test_dap_session` is that smoke test. It drives +the same `gdb -i dap` command without VS Code: it sets a breakpoint in a +JIT input file, and checks the stack and the variables. + +## Without VS Code + +```sh +CPPJIT_DEBUGGER_SUPPORT=1 gdb -iex "source $(python -c 'import cppjit.debugger as d; print(d.gdb_script())')" \ + -ex 'break -qualified summarize' -ex run --args python demo.py +``` + +`bt` shows the Python frames between the C++ frames. +`cppjit sources [REGEX]` lists the JIT inputs, or the lines that match +REGEX as `FILE:LINE`. `cppjit py-bt` prints the Python stack. +`cppjit gil` shows the thread that holds the GIL. gdb also loads the +extension by itself with `add-auto-load-safe-path `. + +Use `break -qualified NAME` for a global function: `break NAME` also +matches each class member with that name, for example in LLVM. + +## Limits + +- Python variables are read only. There is no evaluation of Python + expressions, only of names. Objects with `__slots__` show no + attributes. C++ types that the debug information does not have show as + Python objects. +- Python breakpoints and Python steps need a launch, not an attach. +- A breakpoint or a step on a line that already ran can need up to 0.2 s + of run time to take effect: CPython stops the line events of lines that + had no breakpoint, and a thread turns them on again. +- In the CLI, a Python frame shows the address of the eval loop that runs + it, and frames without debug information show `at LIBRARY` in place of + `from LIBRARY` (gdb prints all frames that way when a frame filter is + active). diff --git a/examples/vscode-gdb/demo.h b/examples/vscode-gdb/demo.h new file mode 100644 index 00000000..b959eafb --- /dev/null +++ b/examples/vscode-gdb/demo.h @@ -0,0 +1,47 @@ +// User C++ that demo.py loads with cppjit.include: a breakpoint in this file +// binds, and a step goes through it, because the JIT compiles it from disk. +#pragma once + +#include +#include +#include + +namespace demo { + +// Counts values into equal-width buckets between lo and hi. +class Histogram { +public: + Histogram(double lo, double hi, std::size_t buckets) + : lo_(lo), hi_(hi), counts_(buckets, 0) {} + + void add(double x) { + std::size_t i = index(x); + counts_[i] += 1; + total_ += 1; + } + + std::size_t count(std::size_t i) const { return counts_.at(i); } + std::size_t total() const { return total_; } + + std::string bar(std::size_t i, char mark = '#') const { + return std::string(counts_.at(i), mark); + } + +private: + std::size_t index(double x) const { + double width = (hi_ - lo_) / counts_.size(); + long i = static_cast((x - lo_) / width); + if (i < 0) + i = 0; + if (i >= static_cast(counts_.size())) + i = static_cast(counts_.size()) - 1; + return static_cast(i); + } + + double lo_; + double hi_; + std::vector counts_; + std::size_t total_ = 0; +}; + +} // namespace demo diff --git a/examples/vscode-gdb/demo.py b/examples/vscode-gdb/demo.py new file mode 100644 index 00000000..47898cd4 --- /dev/null +++ b/examples/vscode-gdb/demo.py @@ -0,0 +1,50 @@ +"""JIT-compile C++ and call it from Python; debug both in VS Code (see README.md).""" + +import os + +import cppjit + +# demo.h is user C++ on disk: the JIT compiles it, so breakpoints and steps work in it. +cppjit.add_include_path(os.path.dirname(os.path.abspath(__file__))) +cppjit.include("demo.h") + +cppjit.cppdef(r""" +#include + +struct Stats { + double mean; + double max; +}; + +Stats summarize(const std::vector& xs) { + Stats stats{0.0, xs.empty() ? 0.0 : xs[0]}; + for (double x : xs) { + stats.mean += x; + if (x > stats.max) + stats.max = x; + } + if (!xs.empty()) + stats.mean /= xs.size(); + return stats; +} +""") + + +def report(values): + stats = cppjit.gbl.summarize(cppjit.gbl.std.vector["double"](values)) + print("mean=%.2f max=%.2f" % (stats.mean, stats.max)) + + +def histogram(values): + hist = cppjit.gbl.demo.Histogram(0.0, 6.0, 3) + for value in values: + hist.add(value) + for i in range(3): + print("bucket %d: %s" % (i, hist.bar(i))) + return hist.total() + + +if __name__ == "__main__": + values = [3.0, 1.5, 4.0, 1.0, 5.5] + report(values) + print("total", histogram(values)) diff --git a/examples/vscode-gdb/extension/dap-proxy.js b/examples/vscode-gdb/extension/dap-proxy.js new file mode 100644 index 00000000..3c61a5fe --- /dev/null +++ b/examples/vscode-gdb/extension/dap-proxy.js @@ -0,0 +1,237 @@ +// A DAP proxy in front of gdb's DAP server that adds Python line breakpoints +// and steps through Python lines. +// usage: node dap-proxy.js [gdb args...] +// +// gdb cannot bind a breakpoint in a .py file. The proxy answers those +// setBreakpoints requests itself and writes the lines to a JSON file. The +// launch request passes the file to the process in CPPJIT_PY_BREAKPOINTS; +// cppjit.debugger then calls a native function at each of these lines, and the +// cppjit gdb extension stops there. +// +// At a stop on a Python line, Step Over, Step Into and Step Out become +// "cppjit py-step" and a continue; Step Into also goes into the C++ code +// that the line calls. A native step that ends in internal code (the gdb +// extension marks its frame "[internal]"), for example at the return from +// JIT code, goes on to the Python line that called it. +// +// The gdb extension gives the variables of Python frames. In a Python frame, +// the proxy answers hover, watch and console expressions that are a Python +// name, such as "self.items[0]", from these variables. +"use strict"; + +const childProcess = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const ENV = "CPPJIT_PY_BREAKPOINTS"; +// Out of the range of gdb's own breakpoint numbers and of the client's sequence numbers. +const FIRST_ID = 1000000; +const FIRST_OWN_SEQ = 1000000000; +const STEPS = { next: "over", stepIn: "into", stepOut: "out" }; +const PYTHON_FRAME = "[py] "; +const INTERNAL_FRAME = "[internal] "; +// The gdb type of the values of Python variables; not useful to the user. +const PYTHON_VALUE_TYPE = "volatile unsigned long"; +const PYTHON_NAME = /^[A-Za-z_]\w*(\.[A-Za-z_]\w*|\[\d+\])*$/; + +// Parses Content-Length framed DAP messages from a stream. +function reader(stream, onMessage) { + let buffer = Buffer.alloc(0); + stream.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + for (;;) { + const end = buffer.indexOf("\r\n\r\n"); + if (end < 0) { + return; + } + const match = /Content-Length:\s*(\d+)/i.exec(buffer.subarray(0, end).toString()); + const length = match ? Number(match[1]) : 0; + if (buffer.length < end + 4 + length) { + return; + } + const body = buffer.subarray(end + 4, end + 4 + length).toString(); + buffer = buffer.subarray(end + 4 + length); + onMessage(JSON.parse(body)); + } + }); +} + +function write(stream, message) { + const data = Buffer.from(JSON.stringify(message)); + stream.write(`Content-Length: ${data.length}\r\n\r\n`); + stream.write(data); +} + +function isPython(source) { + return Boolean(source && source.path && source.path.endsWith(".py")); +} + +function main(argv) { + const gdb = childProcess.spawn(argv[0], argv.slice(1), { stdio: ["pipe", "pipe", "inherit"] }); + const file = path.join(os.tmpdir(), `cppjit-py-breakpoints-${process.pid}.json`); + const lines = {}; + // The name of the top frame of each thread at its last stop. + const topFrame = new Map(); + // The name of each frame id in the stack traces of the current stop. + const frameNames = new Map(); + // Threads whose next stop ends a Python step. + const stepping = new Set(); + // Callbacks for the responses to the proxy's own requests. + const pending = new Map(); + let seq = 0; + let ownSeq = FIRST_OWN_SEQ; + let nextId = FIRST_ID; + let attached = false; + + const toClient = (message) => write(process.stdout, { ...message, seq: ++seq }); + const respond = (request, body) => + toClient({ type: "response", request_seq: request.seq, command: request.command, success: true, body }); + const ask = (command, args) => + new Promise((resolve) => { + const message = { seq: ++ownSeq, type: "request", command, arguments: args }; + pending.set(message.seq, resolve); + write(gdb.stdin, message); + }); + const save = () => { + fs.writeFileSync(file + ".tmp", JSON.stringify(lines)); + fs.renameSync(file + ".tmp", file); + }; + save(); + + const pythonStep = async (threadId, mode) => { + stepping.add(threadId); + await ask("evaluate", { expression: `cppjit py-step --arm ${mode}`, context: "repl" }); + await ask("continue", { threadId }); + }; + + const child = async (variablesReference, name) => { + const answer = await ask("variables", { variablesReference }); + return ((answer.success && answer.body.variables) || []).find((v) => v.name === name); + }; + + const pythonVariable = async (frameId, expression) => { + const parts = expression.match(/[A-Za-z_]\w*|\d+/g); + const answer = await ask("scopes", { frameId }); + let found; + for (const scope of (answer.success && answer.body.scopes) || []) { + if (scope.name !== "Registers" && scope.variablesReference) { + found = await child(scope.variablesReference, parts[0]); + if (found) { + break; + } + } + } + for (const part of parts.slice(1)) { + if (!found || !found.variablesReference) { + return undefined; + } + found = await child(found.variablesReference, part); + } + return found; + }; + + const evaluate = async (request) => { + const args = request.arguments; + const found = await pythonVariable(args.frameId, args.expression.trim()); + if (!found) { + write(gdb.stdin, request); + return; + } + const { value, type, ...rest } = found; + respond(request, { ...rest, result: value }); + }; + + const onStopped = async (event) => { + const threadId = event.body.threadId; + frameNames.clear(); + const answer = await ask("stackTrace", { threadId, startFrame: 0, levels: 1 }); + const frames = (answer.success && answer.body.stackFrames) || []; + const top = frames.length ? frames[0].name : ""; + if (event.body.reason === "step" && top.startsWith(INTERNAL_FRAME)) { + // The step left the user's C++ code: go on to the Python caller. + await pythonStep(threadId, "return"); + return; + } + topFrame.set(threadId, top); + // gdb's internal breakpoint for Python lines has a negative number. + const hits = event.body.hitBreakpointIds || []; + const own = hits.filter((id) => id > 0); + const body = { ...event.body, hitBreakpointIds: own }; + if (stepping.delete(threadId)) { + body.reason = "step"; + } else if (own.length < hits.length) { + body.reason = "breakpoint"; + } + toClient({ ...event, body }); + }; + + reader(process.stdin, (message) => { + const args = message.arguments || {}; + if (message.type === "request" && message.command === "launch") { + const env = { ...(args.env || {}), [ENV]: file, PYTHONUNBUFFERED: "1" }; + message = { ...message, arguments: { ...args, env } }; + } else if (message.type === "request" && message.command === "attach") { + attached = true; + } else if (message.type === "request" && message.command === "setBreakpoints" && isPython(args.source)) { + const wanted = args.breakpoints || []; + lines[path.resolve(args.source.path)] = wanted.map((b) => b.line); + save(); + respond(message, { + breakpoints: wanted.map((b) => attached + ? { verified: false, line: b.line, source: args.source, + message: "Python breakpoints need a launch with CPPJIT_DEBUGGER_SUPPORT=1" } + : { id: nextId++, verified: true, line: b.line, source: args.source }), + }); + return; + } else if (message.type === "request" && message.command === "evaluate" && + (frameNames.get(args.frameId) || "").startsWith(PYTHON_FRAME) && + PYTHON_NAME.test((args.expression || "").trim())) { + evaluate(message); + return; + } else if (message.type === "request" && message.command in STEPS && + (topFrame.get(args.threadId) || "").startsWith(PYTHON_FRAME)) { + respond(message, {}); + pythonStep(args.threadId, STEPS[message.command]); + return; + } + write(gdb.stdin, message); + }); + + reader(gdb.stdout, (message) => { + if (message.type === "response" && message.success && message.command === "stackTrace") { + for (const frame of message.body.stackFrames || []) { + frameNames.set(frame.id, frame.name); + } + } else if (message.type === "response" && message.success && message.command === "variables") { + for (const variable of message.body.variables || []) { + if (variable.type === PYTHON_VALUE_TYPE) { + delete variable.type; + } + } + } + if (message.type === "response" && pending.has(message.request_seq)) { + const resolve = pending.get(message.request_seq); + pending.delete(message.request_seq); + resolve(message); + } else if (message.type === "event" && message.event === "stopped" && message.body.threadId !== undefined) { + onStopped(message); + } else if (message.type === "event" && message.event === "breakpoint" && message.body.breakpoint.id < 0) { + // Internal to the gdb extension: not a breakpoint of the user. + } else { + toClient(message); + } + }); + + const cleanUp = () => fs.rmSync(file, { force: true }); + gdb.on("exit", (code) => { + cleanUp(); + process.exit(code === null ? 1 : code); + }); + process.stdin.on("end", () => gdb.stdin.end()); + for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => gdb.kill(signal)); + } +} + +main(process.argv.slice(2)); diff --git a/examples/vscode-gdb/extension/extension.js b/examples/vscode-gdb/extension/extension.js new file mode 100644 index 00000000..dfdb53ee --- /dev/null +++ b/examples/vscode-gdb/extension/extension.js @@ -0,0 +1,61 @@ +// Runs gdb's own DAP server (gdb -i dap, gdb 16 or later) with the cppjit gdb extension loaded, +// behind dap-proxy.js, which adds Python line breakpoints. +"use strict"; + +const vscode = require("vscode"); +const childProcess = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const TYPE = "cppjit-gdb"; +// find_spec does not import cppjit, so no JIT starts. +const FIND_SCRIPT = + "import importlib.util as u, os; " + + "print(os.path.join(u.find_spec('cppjit').submodule_search_locations[0], 'debugger', 'gdb', 'cppjit_gdb.py'))"; + +function onPath(program, env) { + if (program.includes(path.sep)) { + return program; + } + for (const dir of (env.PATH || "").split(path.delimiter)) { + const candidate = path.join(dir, program); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return program; +} + +function activate(context) { + context.subscriptions.push( + vscode.debug.registerDebugConfigurationProvider(TYPE, { + resolveDebugConfiguration(folder, config) { + // gdb's launch request replaces the whole environment of the process. + config.env = { ...process.env, ...(config.env || {}) }; + config.program = onPath(config.program || "python3", config.env); + return config; + }, + }), + vscode.debug.registerDebugAdapterDescriptorFactory(TYPE, { + createDebugAdapterDescriptor(session) { + const config = session.configuration; + const script = + config.cppjitGdbScript || + childProcess.execFileSync(config.program, ["-c", FIND_SCRIPT], { env: config.env, encoding: "utf8" }).trim(); + // The extension host's own Node runs the proxy; Electron needs ELECTRON_RUN_AS_NODE for that. + return new vscode.DebugAdapterExecutable( + process.execPath, + [ + path.join(__dirname, "dap-proxy.js"), config.gdb || "gdb", "-i", "dap", "-iex", `source ${script}`, + "-iex", `set cppjit internal-frames ${config.internalFrames === "show" ? "show" : "hide"}`, + ], + { env: { ELECTRON_RUN_AS_NODE: "1" } }, + ); + }, + }), + ); +} + +function deactivate() {} + +module.exports = { activate, deactivate }; diff --git a/examples/vscode-gdb/extension/package.json b/examples/vscode-gdb/extension/package.json new file mode 100644 index 00000000..ee762936 --- /dev/null +++ b/examples/vscode-gdb/extension/package.json @@ -0,0 +1,124 @@ +{ + "name": "cppjit-gdb", + "displayName": "cppjit gdb (example)", + "description": "Debug Python processes that JIT-compile C++ with cppjit, through gdb's DAP server.", + "publisher": "cppjit-examples", + "version": "0.0.7", + "license": "BSD-3-Clause-LBNL", + "engines": { + "vscode": "^1.80.0" + }, + "categories": [ + "Debuggers" + ], + "main": "./extension.js", + "activationEvents": [ + "onDebugResolve:cppjit-gdb" + ], + "contributes": { + "breakpoints": [ + { + "language": "cpp" + }, + { + "language": "c" + }, + { + "language": "python" + } + ], + "debuggers": [ + { + "type": "cppjit-gdb", + "label": "cppjit (gdb)", + "languages": [ + "cpp", + "python" + ], + "configurationAttributes": { + "launch": { + "required": [ + "program" + ], + "properties": { + "program": { + "type": "string", + "default": "python3", + "description": "The Python interpreter." + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "cwd": { + "type": "string" + }, + "env": { + "type": "object", + "description": "Added to the environment of VS Code." + }, + "stopAtBeginningOfMainSubprogram": { + "type": "boolean", + "default": false + }, + "gdb": { + "type": "string", + "default": "gdb", + "description": "gdb 16 or later." + }, + "cppjitGdbScript": { + "type": "string", + "description": "Path of cppjit/debugger/gdb/cppjit_gdb.py. The default asks the Python interpreter." + }, + "internalFrames": { + "type": "string", + "enum": [ + "hide", + "show" + ], + "default": "hide", + "description": "Show the frames of cppjit, CppInterOp and CPython in the call stack, and step through them." + } + } + }, + "attach": { + "required": [ + "pid" + ], + "properties": { + "pid": { + "type": [ + "number", + "string" + ] + }, + "program": { + "type": "string", + "default": "python3", + "description": "The Python interpreter." + }, + "gdb": { + "type": "string", + "default": "gdb" + }, + "cppjitGdbScript": { + "type": "string" + }, + "internalFrames": { + "type": "string", + "enum": [ + "hide", + "show" + ], + "default": "hide", + "description": "Show the frames of cppjit, CppInterOp and CPython in the call stack, and step through them." + } + } + } + } + } + ] + } +} diff --git a/python/cppjit-debugger/__init__.py b/python/cppjit-debugger/__init__.py new file mode 100644 index 00000000..1ae7b653 --- /dev/null +++ b/python/cppjit-debugger/__init__.py @@ -0,0 +1,455 @@ +"""Source-level debugging of JIT-compiled C++ with gdb (experimental). + +cppjit ships this package as ``cppjit.debugger`` only when it is built with +``-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON``. Set ``CPPJIT_DEBUGGER_SUPPORT=1`` +before ``import cppjit`` to start a debug session: + +- The interpreter compiles all C++ with ``-g -O0``. +- Each C++ input (``cppdef``, ``include``, ...) goes to a file in + ``source_dir()``, so gdb shows the source and binds ``file:line`` + breakpoints. File names are hashes of the input, so they stay the same + between runs. ``CPPINTEROP_JIT_SOURCE_DIR`` overrides the directory. +- Python source that exists only in memory (``python -c``, notebook cells) + goes to files there too, so mixed backtraces can show it. +- Python and C stdout are line buffered, so output shows before each stop. + +Load the gdb extension with ``source `` in gdb, or let gdb +auto-load it with ``add-auto-load-safe-path ``. + +Python line breakpoints: if ``CPPJIT_PY_BREAKPOINTS`` names a JSON file +``{"": [line, ...]}``, each of these lines calls the native function +``cppjit_debug_break_python`` before it runs, and the gdb extension stops +there. The process reads the file again when it changes. The debugger can +also ask for a step to the next Python line (``cppjit py-step`` in gdb). +This needs ``sys.monitoring`` (Python 3.12 or later). +""" + +from __future__ import annotations + +import hashlib +import json +import linecache +import os +import sys +import threading +import time +from types import CodeType, FrameType, ModuleType +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + import ctypes + from _ctypes import CFuncPtr + +__all__ = ["enabled", "source_dir", "gdb_script", "write_python_source"] + +_PYTHON_INDEX = "python-sources-%d.json" +_BREAKPOINTS = "CPPJIT_PY_BREAKPOINTS" +_POLL_SECONDS = 0.2 +_written: dict[str, tuple[str, str]] = {} + + +def enabled() -> bool: + """Whether this process runs a debug session (``CPPJIT_DEBUGGER_SUPPORT``).""" + return os.environ.get("CPPJIT_DEBUGGER_SUPPORT", "") not in ("", "0") + + +def source_dir() -> str | None: + """The directory that receives the JIT inputs, or None.""" + return os.environ.get("CPPINTEROP_JIT_SOURCE_DIR") or None + + +def gdb_script() -> str: + """The path of the gdb extension script.""" + return os.path.join(os.path.dirname(__file__), "gdb", "cppjit_gdb.py") + + +def _default_source_dir() -> str: + # The same as setDefaultSourceDir in libcppjit, which sets it first in C. + return os.path.join( + os.environ.get("TMPDIR") or "/tmp", "cppjit-debug-%d" % os.getuid() + ) + + +def write_python_source(filename: str, source: str | bytes) -> str | None: + """Write SOURCE, the Python code compiled as FILENAME, to the source dir + if FILENAME is not a file. Return the path written, or None.""" + directory = source_dir() + if not directory or not source or os.path.isfile(filename): + return None + if isinstance(source, bytes): + source = source.decode("utf-8", "replace") + digest = hashlib.sha256(source.encode()).hexdigest()[:16] + if _written.get(filename, (None,))[0] == digest: + return _written[filename][1] + base = "".join( + c if c.isalnum() or c in "._-" else "_" for c in os.path.basename(filename) + ) + path = os.path.join( + directory, "py_%s_%s.py" % (digest, base.strip("_") or "snippet") + ) + try: + os.makedirs(directory, mode=0o700, exist_ok=True) + with open(path, "w") as f: + f.write(source) + _written[filename] = (digest, path) + # The gdb extension maps code object file names to files through this index. + index = os.path.join(directory, _PYTHON_INDEX % os.getpid()) + with open(index + ".tmp", "w") as f: + json.dump({name: entry[1] for name, entry in _written.items()}, f) + os.replace(index + ".tmp", index) + except OSError: + return None + return path + + +def _on_audit(event: str, args: tuple[Any, ...]) -> None: + if event != "compile": + return + try: + source, filename = args[0], args[1] + if not isinstance(filename, str) or filename.startswith(" tuple[str, ...]: + import sysconfig + + paths = sysconfig.get_paths() + dirs = {os.path.realpath(paths[k]) for k in ("stdlib", "platstdlib") if k in paths} + # A directory of symlinks, as in a nix profile, has another real path than its files. + dirs.add(os.path.dirname(os.path.realpath(os.__file__))) + dirs.add(os.path.realpath(os.path.dirname(os.path.dirname(__file__)))) + return tuple(d + os.sep for d in dirs) + + +class _PythonBreakpoints: + """Line breakpoints and line steps in Python code, for a native debugger.""" + + def __init__(self, path: str, monitoring: ModuleType, native: _Native) -> None: + self._path = path + self._mon = monitoring + self._native = native + self._generation: int | None = None + self._stamp: int | None = None + self._lines: dict[str, frozenset[int]] = {} + self._files: dict[str, str] = {} + self._library = _library_dirs() + self._tool: int = monitoring.DEBUGGER_ID + self._reload_lock = threading.Lock() + self._poller: int | None = None + # The step mode that the events were restarted for, and the frame depth it began at. + self._armed = 0 + self._step_depth = 0 + # The caller that a step stops in after a return: its code, and its depth. + self._return_to: tuple[CodeType, int] | None = None + + def start(self) -> bool: + try: + self._mon.use_tool_id(self._tool, "cppjit") + except ValueError: + return False # Another debugger, such as debugpy, has the tool slot. + self.reload() + self._native.on_load(self._on_leave_jit) + events = self._mon.events + self._mon.register_callback(self._tool, events.LINE, self._on_line) + self._mon.register_callback(self._tool, events.PY_RETURN, self._on_return) + self._mon.register_callback( + self._tool, events.INSTRUCTION, self._on_instruction + ) + self._mon.set_events(self._tool, self._mon.events.LINE) + poller = threading.Thread( + target=self._poll, name="cppjit-py-breakpoints", daemon=True + ) + poller.start() + self._poller = poller.ident + return True + + def reload(self) -> bool: + """Read the file if it changed; return True if it did.""" + # The poll thread and the line events both reload; a line must not + # see the new stamp with the old lines. + with self._reload_lock: + return self._reload() + + def _reload(self) -> bool: + try: + stamp = os.stat(self._path).st_mtime_ns + except OSError: + stamp = None + if stamp == self._stamp: + return False + self._stamp = stamp + try: + with open(self._path) as f: + data = json.load(f) + except (OSError, ValueError): + data = {} + self._lines = { + os.path.realpath(name): frozenset(lines) for name, lines in data.items() + } + return True + + def _poll(self) -> None: + while True: + time.sleep(_POLL_SECONDS) + # A step that began in native code finds no Python code to arm it. + stepping = self._native.step_mode() and not self._armed + if self.reload() or stepping: + # Lines that returned DISABLE get their events back. + self._mon.restart_events() + if stepping: + self._armed = self._native.step_mode() + + def _path_of(self, code: CodeType) -> str: + name = code.co_filename + path = self._files.get(name) + if path is None: + path = self._files[name] = os.path.realpath(name) + return path + + def _depth(self, frame: FrameType | None) -> int: + depth = 0 + while frame is not None: + depth += 1 + frame = frame.f_back + return depth + + def _step_stops(self, mode: int, path: str, frame: FrameType) -> bool: + if path.startswith(self._library) or path.startswith("<"): + return False + if mode in (_STEP_INTO, _STEP_RETURN): + return True + depth = self._depth(frame) + return ( + depth <= self._step_depth + if mode == _STEP_OVER + else depth < self._step_depth + ) + + def _stop(self, frame: FrameType) -> None: + self._native.clear_step() + self._armed = 0 + self._end_return_step() + self._native.break_python() + # The debugger may have asked for a step during the stop. + mode = self._native.step_mode() + if mode: + self._step_depth = self._depth(frame) + self._armed = mode + if mode in (_STEP_OVER, _STEP_OUT): + events = self._mon.events + self._mon.set_events(self._tool, events.LINE | events.PY_RETURN) + self._mon.restart_events() + + def _end_return_step(self) -> None: + self._mon.set_events(self._tool, self._mon.events.LINE) + if self._return_to is not None: + self._mon.set_local_events(self._tool, self._return_to[0], 0) + self._return_to = None + + def _stepping_here(self) -> int: + return ( + self._native.step_mode() + and self._native.step_thread() == threading.get_native_id() + ) + + def _on_return(self, code: CodeType, offset: int, value: object) -> None: + # A return into the middle of a line has no line event: stop at the + # caller's next instruction. + try: + if not self._stepping_here() or self._return_to is not None: + return None + frame = sys._getframe(1) + if self._depth(frame) != self._step_depth or frame.f_back is None: + return None + caller = frame.f_back + path = self._path_of(caller.f_code) + if path.startswith(self._library) or path.startswith("<"): + return None + self._return_to = (caller.f_code, self._step_depth - 1) + self._mon.set_local_events( + self._tool, caller.f_code, self._mon.events.INSTRUCTION + ) + except Exception: + self._fail() + return None + + def _on_leave_jit(self) -> None: + # A step out of C++ code: stop in the Python frame that called it. + try: + if ( + self._native.step_mode() != _STEP_RETURN + or not self._stepping_here() + or self._return_to is not None + ): + return + frame: FrameType | None = sys._getframe(1) + while frame is not None: + path = self._path_of(frame.f_code) + if not (path.startswith(self._library) or path.startswith("<")): + break + frame = frame.f_back + if frame is None: + return + self._return_to = (frame.f_code, self._depth(frame)) + self._mon.set_local_events( + self._tool, frame.f_code, self._mon.events.INSTRUCTION + ) + except Exception: + self._fail() + + def _on_instruction(self, code: CodeType, offset: int) -> None: + try: + target = self._return_to + if target is None or code is not target[0] or not self._stepping_here(): + return None + frame = sys._getframe(1) + if self._depth(frame) == target[1]: + self._stop(frame) + except Exception: + self._fail() + return None + + def _on_line(self, code: CodeType, line: int) -> object: + try: + return self._line_event(code, line) + except Exception: + self._fail() + return None + + def _fail(self) -> None: + # An exception here would go to the user's code; stop watching instead. + self._mon.set_events(self._tool, 0) + import traceback + + print("cppjit.debugger: Python breakpoints are off after:", file=sys.stderr) + traceback.print_exc() + + def _line_event(self, code: CodeType, line: int) -> object: + if threading.get_ident() == self._poller: + return self._mon.DISABLE + generation = self._native.generation() + if generation != self._generation: + # The debugger stopped since the last check: take breakpoints set + # in the stop. Another thread must not see the generation as + # seen before the lines are read. + with self._reload_lock: + changed = self._reload() + self._generation = generation + if changed: + self._mon.restart_events() + path = self._path_of(code) + frame = sys._getframe(2) + mode = self._native.step_mode() + if mode and self._native.step_thread() == threading.get_native_id(): + if line in self._lines.get(path, ()) or self._step_stops(mode, path, frame): + self._stop(frame) + return None # Each line must report while the step runs. + if line not in self._lines.get(path, ()): + return self._mon.DISABLE + self._stop(frame) + return None + + +class _Native: + """The debugger hooks in libcppjit, found once cppjit has loaded it.""" + + _generation: ctypes.c_int + _step: ctypes.c_int + _step_thread: ctypes.c_ulong + + def __init__(self) -> None: + self._lib: ctypes.CDLL | None = None + self._on_leave: Callable[[], None] | None = None + self._hook: CFuncPtr | None = None + + def on_load(self, on_leave: Callable[[], None]) -> None: + """Make the library call ON_LEAVE after each JIT call during a step.""" + self._on_leave = on_leave + + def _loaded(self) -> ctypes.CDLL | None: + if self._lib is None: + # Until its import completes, the module has no spec; loading the + # library first here would also come before cppjit's own preload. + module = sys.modules.get("cppjit.libcppjit") + origin = getattr(getattr(module, "__spec__", None), "origin", None) + if not origin: + return None + import ctypes + + lib = ctypes.CDLL(origin) + lib.cppjit_debug_break_python.restype = None + self._generation = ctypes.c_int.in_dll( + lib, "cppjit_debug_python_generation" + ) + self._step = ctypes.c_int.in_dll(lib, "cppjit_debug_python_step") + self._step_thread = ctypes.c_ulong.in_dll( + lib, "cppjit_debug_python_step_thread" + ) + if self._on_leave is not None: + self._hook = ctypes.CFUNCTYPE(None)(self._on_leave) + ctypes.c_void_p.in_dll( + lib, "cppjit_debug_python_return_hook" + ).value = ctypes.cast(self._hook, ctypes.c_void_p).value + self._lib = lib + return self._lib + + def generation(self) -> int | None: + return self._generation.value if self._loaded() else None + + def step_mode(self) -> int: + return self._step.value if self._loaded() else 0 + + def step_thread(self) -> int: + return self._step_thread.value if self._loaded() else 0 + + def clear_step(self) -> None: + if self._loaded(): + self._step.value = 0 + + def break_python(self) -> None: + lib = self._loaded() + if lib: + lib.cppjit_debug_break_python() + + +def _start_python_breakpoints() -> _PythonBreakpoints | None: + path = os.environ.get(_BREAKPOINTS) + monitoring = getattr(sys, "monitoring", None) + if not path or monitoring is None: + return None + breakpoints = _PythonBreakpoints(path, monitoring, _Native()) + return breakpoints if breakpoints.start() else None + + +def _flush_output() -> None: + for stream in (sys.stdout, sys.stderr): + try: + stream.reconfigure(line_buffering=True, write_through=True) # type: ignore[union-attr] + except (AttributeError, ValueError): + pass # Not a TextIOWrapper, for example a Jupyter stream. + + +def _configure() -> None: + """Prepare the process for a debug session; libcppjit calls this when it loads.""" + if not enabled(): + return + os.environ.setdefault("CPPINTEROP_JIT_SOURCE_DIR", _default_source_dir()) + argv = getattr(sys, "orig_argv", []) + if "-c" in argv[:-1]: + write_python_source("", argv[argv.index("-c") + 1]) + sys.addaudithook(_on_audit) + # Output must show at once when a debugger stops the process. + _flush_output() + _start_python_breakpoints() diff --git a/python/cppjit-debugger/gdb/__init__.py b/python/cppjit-debugger/gdb/__init__.py new file mode 100644 index 00000000..230225cd --- /dev/null +++ b/python/cppjit-debugger/gdb/__init__.py @@ -0,0 +1,28 @@ +"""gdb extension for processes that run cppjit. + +Load it in gdb with ``source /cppjit_gdb.py``; gdb also +auto-loads it with libcppjit.so if the package directory is in its +auto-load safe path. It adds: + +- a frame filter (``cppjit-python``) that shows the Python frames of each + thread between its native frames, with their arguments and locals; +- ``cppjit sources``: the C++ inputs that the JIT compiled from files; +- ``cppjit py-bt``: the Python backtrace of the selected thread; +- ``cppjit gil``: the thread that holds the GIL; +- a stop at each Python line breakpoint of ``cppjit.debugger``; +- FILE:LINE breakpoints that wait for the JIT to compile the line. + +Only cpython.py is free of gdb imports. +""" + +from __future__ import annotations + + +def install() -> None: + from . import breakpoints, commands, frames, lines, values + + frames.install() + values.install() + commands.install() + breakpoints.install() + lines.install() diff --git a/python/cppjit-debugger/gdb/breakpoints.py b/python/cppjit-debugger/gdb/breakpoints.py new file mode 100644 index 00000000..0afea9b9 --- /dev/null +++ b/python/cppjit-debugger/gdb/breakpoints.py @@ -0,0 +1,223 @@ +"""Stops at Python line breakpoints, and steps from Python into C++. + +cppjit.debugger calls the native function frames.PYTHON_BREAK when a Python +line in CPPJIT_PY_BREAKPOINTS is about to run, or at the end of a Python +step. An internal breakpoint there stops the process; the frame filter +then shows the Python line as the top frame. + +During a step into, libcppjit calls ENTER_JIT with the Cpp::JitCall of each +call into JIT code. An internal breakpoint there finds the functions that +the JitCall's wrapper calls, and stops at the first one that runs. +""" + +from __future__ import annotations + +import re + +import gdb + +from . import cpython, frames, inferior +from .frames import DEBUG_MODULE, PYTHON_BREAK + +ENTER_JIT = "cppjit_debug_enter_jit" +# The first argument register of each architecture. +_ARGUMENT = {"i386:x86-64": "rdi", "aarch64": "x0"} +_CALL = re.compile(r"\bcall\w*\s+(?:\*\S+\s+#\s+)?(0x[0-9a-f]+)") +_JUMP = re.compile(r"\bjmp\w*\s+\*\S+\s+#\s+(0x[0-9a-f]+)") +_LIBRARY = ("std::", "__gnu_cxx::", "__cxxabiv1::") +_breakpoints: dict[int, tuple[_PythonBreak, _EnterJitBreak]] = {} +# The gdb thread numbers that step into C++, and the breakpoints of the step. +_entering: set[int] = set() +_entries: list[_EntryBreak] = [] + + +class _PythonBreak(gdb.Breakpoint): + """Reports the Python line, not the internal breakpoint.""" + + def __init__(self) -> None: + super().__init__(PYTHON_BREAK, internal=True) + self.silent = True + + def stop(self) -> bool: + try: + frames = [f for s in inferior.python_segments() for f in s] + except (cpython.Unsupported, gdb.error): + frames = [] + for frame in frames: + if not frame.filename.endswith(DEBUG_MODULE): + print( + "Python breakpoint, %s () at %s:%d" + % (frame.name, inferior.python_file(frame.filename), frame.line) + ) + break + return True + + +def _u64(address: int) -> int: + return int.from_bytes(inferior.read(address, 8), "little") + + +def _function_block(pc: int) -> gdb.Block | None: + try: + block = gdb.block_for_pc(pc) + except RuntimeError: + return None + while block is not None and block.function is None: + block = block.superblock + return block + + +def _resolve(target: int, architecture: gdb.Architecture) -> int: + """The function that a call to TARGET runs: TARGET, or the target of its jump stub.""" + if _function_block(target) is None: + found = _JUMP.search(architecture.disassemble(target)[0]["asm"]) + if found: + target = _u64(int(found.group(1), 16)) + return target + + +def _called_functions( + wrapper: int, architecture: gdb.Architecture +) -> list[tuple[str, int]]: + """The user functions with debug information that the code at WRAPPER calls.""" + block = _function_block(wrapper) + if block is None: + return [] + found: list[tuple[str, int]] = [] + for instruction in architecture.disassemble(block.start, block.end - 1): + call = _CALL.search(instruction["asm"]) + if not call: + continue + target = int(call.group(1), 16) + if "*" in instruction["asm"].split("#")[0]: + target = _u64(target) # call through a pointer in memory + target = _resolve(target, architecture) + callee = _function_block(target) + if callee is None or callee.function.symtab is None: + continue + name = callee.function.name + if name.startswith(frames.WRAPPER_PREFIX) or target in [t for _, t in found]: + continue + found.append((name, target)) + users = [f for f in found if not f[0].startswith(_LIBRARY)] + return users or found + + +class _EntryBreak(gdb.Breakpoint): + """The first line of a C++ function that a Python step goes into.""" + + def __init__(self, name: str, address: int, thread: int) -> None: + try: + super().__init__("-qualified " + name, internal=True, temporary=True) + except gdb.error: + super().__init__("*0x%x" % address, internal=True, temporary=True) + self.thread = thread + + def stop(self) -> bool: + # The step ends here, not at the next Python line. + try: + step = int(gdb.parse_and_eval("(unsigned long)&cppjit_debug_python_step")) + gdb.selected_inferior().write_memory(step, b"\0\0\0\0") + except gdb.error: + pass + return True + + +class _EnterJitBreak(gdb.Breakpoint): + """Finds the C++ functions that a JIT call runs, during a step into.""" + + def __init__(self) -> None: + super().__init__(ENTER_JIT, internal=True) + self.silent = True + + def stop(self) -> bool: + thread = gdb.selected_thread() + if thread is None or thread.num not in _entering: + return False + _entering.discard(thread.num) + try: + frame = gdb.newest_frame() + architecture = frame.architecture() + register = _ARGUMENT.get(architecture.name()) + if register is None: + return False + # Cpp::JitCall starts with the union of its wrapper function pointers. + wrapper = _u64(int(frame.read_register(register))) + for name, address in _called_functions(wrapper, architecture): + _entries.append(_EntryBreak(name, address, thread.num)) + except (gdb.error, RuntimeError): + pass + return False + + +def _arm(objfile: gdb.Objfile) -> None: + inferior = gdb.selected_inferior() + if inferior.num in _breakpoints or not objfile.filename.endswith("libcppjit.so"): + return + _breakpoints[inferior.num] = (_PythonBreak(), _EnterJitBreak()) + + +def _on_new_objfile(event: gdb.NewObjFileEvent) -> None: + _arm(event.new_objfile) + + +def _set_int(symbol: str, value: int | str) -> bool: + """Writes VALUE to the libcppjit variable SYMBOL; False if it is not loaded.""" + try: + gdb.execute( + "set var *(%s)&%s = %s" % (_TYPES[symbol], symbol, value), to_string=True + ) + except gdb.error: + return False + return True + + +_TYPES = { + "cppjit_debug_python_generation": "int*", + "cppjit_debug_python_step": "int*", + "cppjit_debug_python_step_thread": "unsigned long*", +} + + +def _on_stop(event: gdb.StopEvent) -> None: + # cppjit.debugger reads breakpoints again when the generation changes. + _set_int( + "cppjit_debug_python_generation", "*(int*)&cppjit_debug_python_generation + 1" + ) + _entering.clear() + while _entries: + entry = _entries.pop() + if entry.is_valid(): + entry.delete() + + +# "return" goes on from C++ code to the Python frame that called it. +STEP_MODES = {"into": 1, "over": 2, "out": 3, "return": 4} + + +def request_step(mode: str) -> None: + """Make the selected thread stop at its next Python line (MODE: into, over, out).""" + thread = gdb.selected_thread() + if thread is None: + raise gdb.GdbError("No thread.") + if not ( + _set_int("cppjit_debug_python_step_thread", thread.ptid[1]) + and _set_int("cppjit_debug_python_step", STEP_MODES[mode]) + ): + raise gdb.GdbError("The process has not loaded libcppjit with Python stepping.") + if mode == "into": + _entering.add(thread.num) + + +def _on_exited(event: gdb.ExitedEvent) -> None: + for breakpoint in _breakpoints.pop(event.inferior.num, ()): + if breakpoint.is_valid(): + breakpoint.delete() + + +def install() -> None: + gdb.events.new_objfile.connect(_on_new_objfile) + gdb.events.stop.connect(_on_stop) + gdb.events.exited.connect(_on_exited) + for objfile in gdb.objfiles(): + _arm(objfile) diff --git a/python/cppjit-debugger/gdb/commands.py b/python/cppjit-debugger/gdb/commands.py new file mode 100644 index 00000000..ee575d96 --- /dev/null +++ b/python/cppjit-debugger/gdb/commands.py @@ -0,0 +1,184 @@ +"""The cppjit prefix command of gdb.""" + +from __future__ import annotations + +import glob +import os +import re +from typing import Callable + +import gdb + +from . import breakpoints, cpython, frames, inferior + + +def _first_code_line(path: str) -> str: + try: + with open(path, errors="replace") as f: + for line in f: + if line.strip() and not line.startswith("#line"): + return line.strip() + except OSError: + pass + return "" + + +def _inputs() -> list[str]: + directory = inferior.source_dir() + if directory is None: + raise gdb.GdbError( + "CPPINTEROP_JIT_SOURCE_DIR is not set in the process; start it with CPPJIT_DEBUGGER_SUPPORT=1" + ) + return sorted( + glob.glob(os.path.join(directory, "input_*.cpp")), key=os.path.getmtime + ) + + +class CppjitCommand(gdb.Command): + """Commands for debugging cppjit processes. + + cppjit sources, cppjit py-bt, cppjit py-step, cppjit gil, and cppjit up and + down, which skip internal frames.""" + + def __init__(self) -> None: + super().__init__("cppjit", gdb.COMMAND_USER, prefix=True) + + +class SourcesCommand(gdb.Command): + """List the C++ inputs that the JIT compiled from files. + + Usage: cppjit sources [REGEX] + Without REGEX, show each input file and its first code line. With REGEX, + show the matching lines as FILE:LINE, ready for "break FILE:LINE".""" + + def __init__(self) -> None: + super().__init__("cppjit sources", gdb.COMMAND_FILES) + + def invoke(self, arg: str, from_tty: bool) -> None: + pattern = re.compile(arg.strip()) if arg.strip() else None + for path in _inputs(): + name = os.path.basename(path) + if pattern is None: + print("%s %s" % (name, _first_code_line(path)[:80])) + continue + with open(path, errors="replace") as f: + for number, line in enumerate(f, 1): + if pattern.search(line): + print("%s:%d: %s" % (name, number, line.rstrip())) + + +class PyBacktraceCommand(gdb.Command): + """Print the Python frames of the selected thread, most recent call last.""" + + def __init__(self) -> None: + super().__init__("cppjit py-bt", gdb.COMMAND_STACK) + + def invoke(self, arg: str, from_tty: bool) -> None: + try: + segments = inferior.python_segments() + except cpython.Unsupported as exc: + raise gdb.GdbError("No Python state: %s" % exc) + frames = [frame for segment in segments for frame in segment] + if not frames: + print("No Python frames in this thread.") + return + print("Traceback (most recent call last):") + for frame in reversed(frames): + path = inferior.python_file(frame.filename) + print(' File "%s", line %s, in %s' % (path, frame.line, frame.name)) + text = _line_text(path, frame.line) + if text: + print(" " + text) + + +def _line_text(path: str, line: int) -> str: + try: + with open(path, errors="replace") as f: + for number, text in enumerate(f, 1): + if number == line: + return text.strip() + except (OSError, TypeError): + pass + return "" + + +class GilCommand(gdb.Command): + """Show which thread holds the GIL. + + A thread stopped while it holds the GIL blocks all other Python threads of + the process, also when other threads are free to run (non-stop mode).""" + + def __init__(self) -> None: + super().__init__("cppjit gil", gdb.COMMAND_STATUS) + + def invoke(self, arg: str, from_tty: bool) -> None: + try: + holder = inferior.runtime().gil_holder() + except cpython.Unsupported as exc: + raise gdb.GdbError("No Python state: %s" % exc) + if holder is None: + print("This is a free-threaded CPython: it has no GIL.") + elif holder == 0: + print("No thread holds the GIL.") + else: + for thread in gdb.selected_inferior().threads(): + if thread.ptid[1] == holder: + print("Thread %d (LWP %d) holds the GIL." % (thread.num, holder)) + return + print("LWP %d holds the GIL." % holder) + + +class PyStepCommand(gdb.Command): + """Continue to the next Python line of the selected thread. + + Usage: cppjit py-step [--arm] [into|over|out|return] + "into" (the default) stops at the next line in user code, or in the + first C++ function with debug information that a JIT call runs. "over" + stops at the next line in this Python frame or a caller, "out" in a + caller. In C++ code, "return" stops in the Python frame that called the + C++ code, after the call. Lines in the standard library and in cppjit + are not user code. With --arm, set the step but do not continue.""" + + def __init__(self) -> None: + super().__init__("cppjit py-step", gdb.COMMAND_RUNNING) + + def invoke(self, arg: str, from_tty: bool) -> None: + words = arg.split() + arm_only = "--arm" in words + words = [w for w in words if w != "--arm"] + mode = words[0] if words else "into" + if len(words) > 1 or mode not in breakpoints.STEP_MODES: + raise gdb.GdbError("Usage: cppjit py-step [--arm] [into|over|out|return]") + breakpoints.request_step(mode) + if not arm_only: + gdb.execute("continue") + + +class _MoveCommand(gdb.Command): + def __init__( + self, name: str, step: Callable[[gdb.Frame], gdb.Frame | None] + ) -> None: + super().__init__("cppjit " + name, gdb.COMMAND_STACK) + self._step = step + + def invoke(self, arg: str, from_tty: bool) -> None: + count = int(arg) if arg.strip() else 1 + frame = gdb.selected_frame() + while count > 0: + frame = self._step(frame) + if frame is None: + raise gdb.GdbError("No more frames outside cppjit internals.") + if not frames.is_internal_frame(frame): + count -= 1 + frame.select() + gdb.execute("frame") + + +def install() -> None: + CppjitCommand() + PyStepCommand() + _MoveCommand("up", lambda f: f.older()) + _MoveCommand("down", lambda f: f.newer()) + SourcesCommand() + PyBacktraceCommand() + GilCommand() diff --git a/python/cppjit-debugger/gdb/cppjit_gdb.py b/python/cppjit-debugger/gdb/cppjit_gdb.py new file mode 100644 index 00000000..6be6b0f0 --- /dev/null +++ b/python/cppjit-debugger/gdb/cppjit_gdb.py @@ -0,0 +1,26 @@ +"""Load the cppjit gdb extension: ``source`` this file in gdb.""" + +from __future__ import annotations + +import importlib.util +import os +import sys + + +def _load() -> None: + name = "cppjit_gdb_extension" + if name in sys.modules: + return + # Load the directory under its own name: importing the cppjit package would start a JIT in gdb. + here = os.path.dirname(os.path.realpath(__file__)) + spec = importlib.util.spec_from_file_location( + name, os.path.join(here, "__init__.py"), submodule_search_locations=[here] + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + module.install() + + +_load() diff --git a/python/cppjit-debugger/gdb/cpython.py b/python/cppjit-debugger/gdb/cpython.py new file mode 100644 index 00000000..7322d39a --- /dev/null +++ b/python/cppjit-debugger/gdb/cpython.py @@ -0,0 +1,736 @@ +"""Read CPython's thread and frame state from process memory. + +CPython 3.13 and later keep a table of structure offsets (_Py_DebugOffsets) +at the start of _PyRuntime for out-of-process tools. With it, a debugger +finds the Python frames of each thread without CPython debug information +and without code that runs in the stopped process. The table layout is +fixed only within a minor version, so each supported version has an entry +in _LAYOUTS. + +This module does not import gdb: a READ(address, size) -> bytes callback +gives it access to memory. It must run on the Python that gdb embeds. +""" + +from __future__ import annotations + +import struct +from typing import Callable, Iterable, Iterator, Sequence + +COOKIE = b"xdebugpy" + +_COMMON_TAIL_313 = [ + ("tuple_object", ["size", "ob_item", "ob_size"]), + ("list_object", ["size", "ob_item", "ob_size"]), +] +_OBJECTS_TAIL = [ + ("dict_object", ["size", "ma_keys", "ma_values"]), + ("float_object", ["size", "ob_fval"]), + ("long_object", ["size", "lv_tag", "ob_digit"]), + ("bytes_object", ["size", "ob_size", "ob_sval"]), + ("unicode_object", ["size", "state", "length", "asciiobject_size"]), +] + +# (minor version) -> field groups of _Py_DebugOffsets, in order; each field is a uint64. +_LAYOUTS = { + 13: [ + ("runtime_state", ["size", "finalizing", "interpreters_head"]), + ( + "interpreter_state", + [ + "size", + "id", + "next", + "threads_head", + "gc", + "imports_modules", + "sysdict", + "builtins", + "ceval_gil", + "gil_runtime_state", + "gil_runtime_state_enabled", + "gil_runtime_state_locked", + "gil_runtime_state_holder", + ], + ), + ( + "thread_state", + [ + "size", + "prev", + "next", + "interp", + "current_frame", + "thread_id", + "native_thread_id", + "datastack_chunk", + "status", + ], + ), + ( + "interpreter_frame", + ["size", "previous", "executable", "instr_ptr", "localsplus", "owner"], + ), + ( + "code_object", + [ + "size", + "filename", + "name", + "qualname", + "linetable", + "firstlineno", + "argcount", + "localsplusnames", + "localspluskinds", + "co_code_adaptive", + ], + ), + ("pyobject", ["size", "ob_type"]), + ("type_object", ["size", "tp_name", "tp_repr", "tp_flags"]), + ] + + _COMMON_TAIL_313 + + _OBJECTS_TAIL, + 14: [ + ("runtime_state", ["size", "finalizing", "interpreters_head"]), + ( + "interpreter_state", + [ + "size", + "id", + "next", + "threads_head", + "threads_main", + "gc", + "imports_modules", + "sysdict", + "builtins", + "ceval_gil", + "gil_runtime_state", + "gil_runtime_state_enabled", + "gil_runtime_state_locked", + "gil_runtime_state_holder", + "code_object_generation", + "tlbc_generation", + ], + ), + ( + "thread_state", + [ + "size", + "prev", + "next", + "interp", + "current_frame", + "thread_id", + "native_thread_id", + "datastack_chunk", + "status", + ], + ), + ( + "interpreter_frame", + [ + "size", + "previous", + "executable", + "instr_ptr", + "localsplus", + "owner", + "stackpointer", + "tlbc_index", + ], + ), + ( + "code_object", + [ + "size", + "filename", + "name", + "qualname", + "linetable", + "firstlineno", + "argcount", + "localsplusnames", + "localspluskinds", + "co_code_adaptive", + "co_tlbc", + ], + ), + ("pyobject", ["size", "ob_type"]), + ("type_object", ["size", "tp_name", "tp_repr", "tp_flags"]), + ] + + _COMMON_TAIL_313 + + [("set_object", ["size", "used", "table", "mask"])] + + _OBJECTS_TAIL, +} + +# Owner of the entry frame that each C-level call of the eval loop pushes: FRAME_OWNED_BY_CSTACK +# in 3.13, FRAME_OWNED_BY_INTERPRETER in 3.14. Higher owners are not Python frames either. +_ENTRY_OWNER = {13: 3, 14: 3} +# _PyStackRef tag bits on f_executable and the fast locals. +_STACKREF_TAG = {13: 0, 14: 3} +# f_globals and f_locals of _PyInterpreterFrame; the debug offsets do not give them. +_F_GLOBALS, _F_LOCALS = 24, 40 +# co_localspluskinds bits (CO_FAST_*); the argument bits exist from 3.14. +_KIND_ARGS = 0x02 | 0x04 | 0x08 +_KIND_HIDDEN = 0x10 +_KIND_CELL = 0x40 | 0x80 +# Py_TPFLAGS_INLINE_VALUES, Py_TPFLAGS_MANAGED_DICT; instances keep the managed dict pointer at -24. +_INLINE_VALUES, _MANAGED_DICT = 1 << 2, 1 << 4 +_MANAGED_DICT_OFFSET = -24 +# PyTypeObject.tp_basicsize; ht_cached_keys of PyHeapTypeObject after sizeof(PyTypeObject), the +# five method tables and ht_name, ht_slots, ht_qualname. +_TP_BASICSIZE = 32 +_HT_CACHED_KEYS = 8 * (4 + 36 + 3 + 10 + 2) + 24 +# PyTypeObject.tp_base and tp_dict, after tp_flags. +_TP_BASE_AFTER_FLAGS, _TP_DICT_AFTER_FLAGS = 88, 96 +# cppjit's proxies of C++ objects (src/cpyrt): CPPInstance has fObject and fFlags after +# PyObject_HEAD; with kIsExtended, fObject points to an ExtendedData, which starts with the object +# address and has fSmartClass at 32. Each bound namespace is a module "cppjit.gbl.NAME". +_CPP_INSTANCE, _CPP_SCOPE = "cppjit.CPPInstance", "cppjit.CPPScope" +_CPP_MODULE = "cppjit.gbl" +_CPP_IS_EXTENDED, _CPP_IS_REFERENCE, _CPP_IS_SMART = 0x0004, 0x0010, 0x0040 +_CPP_SMART_CLASS = 32 +_MAX_SCOPES = 4096 +# Py_TPFLAGS_*_SUBCLASS bits. +_LONG, _LIST, _TUPLE, _BYTES, _UNICODE, _DICT, _, _TYPE = ( + 1 << n for n in range(24, 32) +) + +_MAX_FRAMES = 4096 +_MAX_STRING = 4096 +_MAX_ITEMS = 1000 +_MAX_DIGITS = 100 +_PREVIEW = 80 + + +class Unsupported(Exception): + """The process has no CPython runtime that this module can read.""" + + +class PyFrame: + """One Python frame: the function's qualified name, its file and its current line. ADDRESS and + CODE are the addresses of the _PyInterpreterFrame and of its code object.""" + + def __init__( + self, name: str, filename: str, line: int, address: int = 0, code: int = 0 + ) -> None: + self.name = name + self.filename = filename + self.line = line + self.address = address + self.code = code + + def __repr__(self) -> str: + return "PyFrame(%r, %r, %r)" % (self.name, self.filename, self.line) + + +def _varint(data: bytes, i: int) -> tuple[int, int]: + b = data[i] + i += 1 + value, shift = b & 63, 0 + while b & 64: + b = data[i] + i += 1 + shift += 6 + value |= (b & 63) << shift + return value, i + + +def _svarint(data: bytes, i: int) -> tuple[int, int]: + value, i = _varint(data, i) + return (-(value >> 1) if value & 1 else value >> 1), i + + +def code_line(linetable: bytes, firstlineno: int, index: int) -> int | None: + """The line of code unit INDEX in a co_linetable (CPython's Objects/locations.md), or None.""" + line, address, i = firstlineno, 0, 0 + while i < len(linetable): + first = linetable[i] + i += 1 + code, length = (first >> 3) & 15, (first & 7) + 1 + here: int | None = line + if code == 15: + here = None + elif code == 14: + delta, i = _svarint(linetable, i) + line += delta + here = line + for _ in range(3): + _, i = _varint(linetable, i) + elif code == 13: + delta, i = _svarint(linetable, i) + line += delta + here = line + elif code >= 10: + line += code - 10 + here = line + i += 2 + else: + i += 1 + if address <= index < address + length: + return here + address += length + return None + + +class Runtime: + """CPython's runtime state at RUNTIME, the address of _PyRuntime. CODE_TYPE is the address of + PyCode_Type, or None to accept any object as a code object.""" + + def __init__( + self, + read: Callable[[int, int], bytes], + runtime: int, + code_type: int | None = None, + ) -> None: + self._read = read + self.address = runtime + self._code_type = code_type + if read(runtime, 8) != COOKIE: + raise Unsupported("no _Py_DebugOffsets cookie at _PyRuntime") + self.version = self._u64(runtime + 8) + self.minor = (self.version >> 16) & 0xFF + if (self.version >> 24) != 3 or self.minor not in _LAYOUTS: + raise Unsupported("CPython %x is not supported" % self.version) + self.free_threaded = bool(self._u64(runtime + 16)) + self.offsets: dict[str, int] = {} + index = 2 + for group, fields in _LAYOUTS[self.minor]: + for field in fields: + index += 1 + self.offsets[group + "." + field] = self._u64(runtime + 8 * index) + # The size of PyObject_HEAD: the fields of other objects come after it. + self._head = self.offsets["pyobject.ob_type"] + 8 + + def _u64(self, address: int) -> int: + return struct.unpack(" int: + return struct.unpack(" int: + return struct.unpack(" int: + return struct.unpack(" int: + return self.offsets[name] + + def string(self, obj: int) -> str: + """The text of the str object at OBJ.""" + state = self._u32(obj + self._off("unicode_object.state")) + length = min(self._i64(obj + self._off("unicode_object.length")), _MAX_STRING) + kind, compact, ascii = (state >> 2) & 7, (state >> 5) & 1, (state >> 6) & 1 + if not compact or length < 0: + return "?" + data = obj + self._off("unicode_object.asciiobject_size") + if not ascii: + # PyCompactUnicodeObject adds utf8_length and utf8 to PyASCIIObject. + data += 16 + raw = self._read(data, length * kind) + return raw.decode( + {1: "latin-1", 2: "utf-16-le", 4: "utf-32-le"}.get(kind, "latin-1"), + "replace", + ) + + def bytes(self, obj: int) -> bytes: + size = min(self._i64(obj + self._off("bytes_object.ob_size")), 1 << 20) + return self._read(obj + self._off("bytes_object.ob_sval"), max(size, 0)) + + def _c_string(self, address: int, limit: int = 256) -> str: + raw = self._read(address, limit) + return ( + raw[: raw.find(b"\0")].decode("utf-8", "replace") if b"\0" in raw else "?" + ) + + def _type(self, obj: int) -> int: + return self._u64(obj + self._off("pyobject.ob_type")) + + def type_name(self, obj: int) -> str: + return self._c_string( + self._u64(self._type(obj) + self._off("type_object.tp_name")) + ) + + def _flags(self, obj: int) -> int: + return self._u64(self._type(obj) + self._off("type_object.tp_flags")) + + def _ref(self, address: int) -> int: + return self._u64(address) & ~_STACKREF_TAG[self.minor] + + def _long(self, obj: int) -> int | None: + tag = self._u64(obj + self._off("long_object.lv_tag")) + count = tag >> 3 + if count > _MAX_DIGITS: + return None + digits = struct.unpack( + "<%dI" % count, + self._read(obj + self._off("long_object.ob_digit"), 4 * count), + ) + value = sum(d << (30 * i) for i, d in enumerate(digits)) + return -value if tag & 3 == 2 else value + + def _sequence(self, obj: int) -> list[int]: + flags = self._flags(obj) + group = "tuple_object" if flags & _TUPLE else "list_object" + size = min(max(self._i64(obj + self._off(group + ".ob_size")), 0), _MAX_ITEMS) + items = obj + self._off(group + ".ob_item") + if not flags & _TUPLE: + items = self._u64(items) + if not size: + return [] + return list(struct.unpack("<%dQ" % size, self._read(items, 8 * size))) + + def _entries( + self, keys: int, values: int, limit: int = _MAX_ITEMS + ) -> list[tuple[int, int]]: + # PyDictKeysObject: dk_log2_index_bytes at 9, dk_kind at 10, dk_nentries at 24, dk_indices at 32. + index_bytes, kind = self._read(keys + 9, 2) + count = min(self._i64(keys + 24), limit) + entries = keys + 32 + (1 << index_bytes) + order: Sequence[int] = range(max(count, 0)) + if values: + # PyDictValues: capacity at 0, size at 1, values at 8, then the insertion order bytes. + capacity, size = self._read(values, 2) + order = self._read(values + 8 + 8 * capacity, min(size, limit)) + items: list[tuple[int, int]] = [] + for i in order: + if kind == 0: + # PyDictKeyEntry: me_hash, me_key, me_value. + key, value = struct.unpack(" list[tuple[int, int]]: + """The (key, value) object addresses of the dict at OBJ, in insertion order.""" + return self._entries( + self._u64(obj + self._off("dict_object.ma_keys")), + self._u64(obj + self._off("dict_object.ma_values")), + limit, + ) + + def attributes(self, obj: int) -> list[tuple[str, int]] | None: + """The (name, object address) of the instance attributes of OBJ, or None.""" + tp, flags = self._type(obj), self._flags(obj) + if not flags & _MANAGED_DICT: + return None + namespace = self._u64(obj + _MANAGED_DICT_OFFSET) + if namespace: + items = self.dict_items(namespace) + elif flags & _INLINE_VALUES: + values = obj + self._i64(tp + _TP_BASICSIZE) + if not self._read(values + 3, 1)[0]: # PyDictValues.valid + return None + keys = self._u64(tp + self._off("type_object.size") + _HT_CACHED_KEYS) + items = self._entries(keys, values) + else: + return None + return [(self.string(k), v) for k, v in items if self._flags(k) & _UNICODE] + + def variables( + self, frame: PyFrame + ) -> tuple[list[tuple[str, int]], list[tuple[str, int]]]: + """The arguments and the local variables of FRAME, a PyFrame: two lists of (name, object + address). Module and class bodies give the names in their namespace dict as locals.""" + code = frame.code + names = self._sequence( + self._u64(code + self._off("code_object.localsplusnames")) + ) + kinds = self.bytes(self._u64(code + self._off("code_object.localspluskinds"))) + base = frame.address + self._off("interpreter_frame.localsplus") + argcount = self._i32(code + self._off("code_object.argcount")) + arguments: list[tuple[str, int]] = [] + local: list[tuple[str, int]] = [] + for i, name in enumerate(names): + kind = kinds[i] if i < len(kinds) else 0 + if kind & _KIND_HIDDEN: + continue + obj = self._ref(base + 8 * i) + if obj and kind & _KIND_CELL and self.type_name(obj) == "cell": + obj = self._u64(obj + self._head) # PyCellObject.ob_ref + if obj: + (arguments if kind & _KIND_ARGS or i < argcount else local).append( + (self.string(name), obj) + ) + if not names: + namespace = self._u64(frame.address + _F_LOCALS) or self._u64( + frame.address + _F_GLOBALS + ) + if namespace and self._flags(namespace) & _DICT: + for key, value in self.dict_items(namespace): + if self._flags(key) & _UNICODE: + text = self.string(key) + if not (text.startswith("__") and text.endswith("__")): + local.append((text, value)) + return arguments, local + + def _name_at(self, obj: int, offset: int) -> str | None: + name = self._u64(obj + offset) + return self.string(name) if name and self._flags(name) & _UNICODE else None + + def _is_subtype(self, tp: int, name: str) -> bool: + base = self._off("type_object.tp_flags") + _TP_BASE_AFTER_FLAGS + for _ in range(64): + if not tp: + return False + if self._c_string(self._u64(tp + self._off("type_object.tp_name"))) == name: + return True + tp = self._u64(tp + base) + return False + + def _is_cpp_scope(self, obj: int) -> int: + return self._flags(obj) & _TYPE and self._is_subtype( + self._type(obj), _CPP_SCOPE + ) + + def cpp_name(self, tp: int) -> str | None: + """The qualified C++ name of the cppjit proxy class TP, or None.""" + interp = next(self.interpreters(), None) + if not interp: + return None + modules = self._u64(interp + self._off("interpreter_state.imports_modules")) + pending: list[tuple[list[str], int]] = [] + for key, scope in self.dict_items(modules, limit=1 << 20): + name = self.string(key) if self._flags(key) & _UNICODE else "" + if name == _CPP_MODULE or name.startswith(_CPP_MODULE + "."): + path = ( + name[len(_CPP_MODULE) + 1 :].split(".") + if name != _CPP_MODULE + else [] + ) + if all(p.isidentifier() for p in path): + pending.append((path, scope)) + seen: set[int] = set() + while pending and len(seen) < _MAX_SCOPES: + path, scope = pending.pop(0) + if scope in seen or not self._is_cpp_scope(scope): + continue + seen.add(scope) + namespace = self._u64( + scope + self._off("type_object.tp_flags") + _TP_DICT_AFTER_FLAGS + ) + if not namespace: + continue + for key, value in self.dict_items(namespace, limit=1 << 16): + if not self._flags(key) & _UNICODE: + continue + if value == tp: + name = self.string(key) + # Template instances have their qualified name as the key. + if path and name.startswith("::".join(path) + "::"): + return name + return "::".join(path + [name]) + pending.append((path + [self.string(key)], value)) + return None + + def cpp_object( + self, obj: int + ) -> tuple[int | None, str, tuple[int, str] | None] | None: + """For a cppjit proxy at OBJ: (address, C++ type name, smart), where smart is the + (address, C++ type name) of the smart pointer that holds the object, or None. None if + OBJ is not a proxy. The address of an object in a smart pointer is None.""" + tp = self._type(obj) + if not self._is_subtype(tp, _CPP_INSTANCE): + return None + flags = self._u32(obj + self._head + 8) + address = self._u64(obj + self._head) + if flags & _CPP_IS_EXTENDED and address: + extended, address = address, self._u64(address) + if flags & _CPP_IS_SMART: + smart = self.cpp_name(self._u64(extended + _CPP_SMART_CLASS)) + name = self.cpp_name(tp) + return ( + (None, name, (address, smart)) + if address and smart and name + else None + ) + elif address and flags & _CPP_IS_REFERENCE: + address = self._u64(address) + name = self.cpp_name(tp) if address else None + return (address, name, None) if name else None + + def summary(self, obj: int, depth: int = 0) -> str: + """A short text for the object at OBJ, like its repr() for the common types.""" + name, flags = self.type_name(obj), self._flags(obj) + if name == "NoneType": + return "None" + if name == "bool": + return "True" if self._long(obj) else "False" + if flags & _LONG: + value = self._long(obj) + return "" if value is None else str(value) + if name == "float": + return repr( + struct.unpack( + "" % self._c_string( + self._u64(obj + self._off("type_object.tp_name")) + ) + if name == "function": + # PyFunctionObject: func_globals, func_builtins, func_name, func_qualname. + return "" % self._name_at(obj, self._head + 24) + if name == "module": + # PyModuleObject: md_dict, md_def, md_state, md_weaklist, md_name. + return "" % self._name_at(obj, self._head + 32) + return "<%s object at 0x%x>" % (name, obj) + + def children(self, obj: int) -> list[tuple[str, int]] | None: + """The items of a list, tuple or dict at OBJ, or the attributes of another object, as + (name, object address); None if it has none to show.""" + flags = self._flags(obj) + if flags & _DICT: + return [(self.summary(k, 1), v) for k, v in self.dict_items(obj)] + if flags & (_LIST | _TUPLE): + return [(str(i), item) for i, item in enumerate(self._sequence(obj))] + if self.type_name(obj) in ("type", "module", "function"): + return None + return self.attributes(obj) or None + + def interpreters(self) -> Iterator[int]: + interp = self._u64(self.address + self._off("runtime_state.interpreters_head")) + while interp: + yield interp + interp = self._u64(interp + self._off("interpreter_state.next")) + + def thread_states(self) -> list[tuple[int, int]]: + """[(PyThreadState address, native thread id)] of every interpreter.""" + found: list[tuple[int, int]] = [] + for interp in self.interpreters(): + tstate = self._u64(interp + self._off("interpreter_state.threads_head")) + while tstate and len(found) < _MAX_FRAMES: + found.append( + ( + tstate, + self._u64(tstate + self._off("thread_state.native_thread_id")), + ) + ) + tstate = self._u64(tstate + self._off("thread_state.next")) + return found + + def thread_state(self, native_thread_id: int) -> int | None: + for tstate, tid in self.thread_states(): + if tid == native_thread_id: + return tstate + return None + + def _frame(self, frame: int) -> PyFrame | None: + executable = ( + self._u64(frame + self._off("interpreter_frame.executable")) + & ~_STACKREF_TAG[self.minor] + ) + if not executable: + return None + if ( + self._code_type is not None + and self._u64(executable + self._off("pyobject.ob_type")) != self._code_type + ): + return None + name = self.string(self._u64(executable + self._off("code_object.qualname"))) + filename = self.string( + self._u64(executable + self._off("code_object.filename")) + ) + firstlineno = self._i32(executable + self._off("code_object.firstlineno")) + instr = self._u64(frame + self._off("interpreter_frame.instr_ptr")) + index = (instr - (executable + self._off("code_object.co_code_adaptive"))) // 2 + line = None + if 0 <= index < (1 << 24): + table = self.bytes( + self._u64(executable + self._off("code_object.linetable")) + ) + line = code_line(table, firstlineno, index) + return PyFrame( + name, + filename, + line if line is not None else firstlineno, + frame, + executable, + ) + + def segments(self, tstate: int) -> list[list[PyFrame]]: + """The Python frames of TSTATE, newest first, split into one list per C-level run of the eval + loop: segment K belongs to the K-th newest _PyEval_EvalFrameDefault native frame.""" + segments: list[list[PyFrame]] = [] + current: list[PyFrame] = [] + frame = self._u64(tstate + self._off("thread_state.current_frame")) + entry = _ENTRY_OWNER[self.minor] + for _ in range(_MAX_FRAMES): + if not frame: + break + owner = self._read(frame + self._off("interpreter_frame.owner"), 1)[0] + if owner == entry: + segments.append(current) + current = [] + elif owner < entry: + py = self._frame(frame) + if py is not None: + current.append(py) + frame = self._u64(frame + self._off("interpreter_frame.previous")) + if current: + segments.append(current) + return segments + + def gil_holder(self) -> int | None: + """The native thread id of the thread that holds the main interpreter's GIL, 0 if the GIL is + free, or None for free-threaded builds.""" + if self.free_threaded: + return None + interp = None + for interp in self.interpreters(): + pass + if interp is None: + return 0 + gil = self._u64(interp + self._off("interpreter_state.ceval_gil")) + base = self._off("interpreter_state.gil_runtime_state") + if not gil or not self._i32( + gil + self._off("interpreter_state.gil_runtime_state_locked") - base + ): + return 0 + holder = self._u64( + gil + self._off("interpreter_state.gil_runtime_state_holder") - base + ) + if not holder: + return 0 + return self._u64(holder + self._off("thread_state.native_thread_id")) + + +def _clip(text: str) -> str: + return text if len(text) <= _PREVIEW else text[: _PREVIEW - 3] + "..." + + +def _join(start: str, parts: Iterable[str], end: str) -> str: + text = start + for i, part in enumerate(parts): + if len(text) > _PREVIEW: + return text + ", ..." + end[-1] + text += (", " if i else "") + part + return text + end diff --git a/python/cppjit-debugger/gdb/frames.py b/python/cppjit-debugger/gdb/frames.py new file mode 100644 index 00000000..4ebc9de6 --- /dev/null +++ b/python/cppjit-debugger/gdb/frames.py @@ -0,0 +1,225 @@ +"""Mixed Python and C++ backtraces. + +The frame filter puts the Python frames that a run of CPython's eval loop +executes in the place of its native _PyEval_EvalFrameDefault frame. Thus +each JIT-compiled C++ frame is shown below the Python line that called it. +The native frame stays available as an elided frame. + +By default, the filter also hides internal frames: the cppjit and CppInterOp +runtime, CPython, the C runtime, the JIT call wrappers, and native code +without line information. If the stop itself is in internal code, the top +frame stays, marked "[internal]". "set cppjit internal-frames show" shows +all frames. +""" + +from __future__ import annotations + +import os +import re +from typing import Iterable, Iterator + +import gdb +from gdb.FrameDecorator import FrameDecorator + +from . import cpython, inferior, values + +FILTER_NAME = "cppjit-python" +_EVAL_LOOP = "_PyEval_EvalFrameDefault" +PYTHON_BREAK = "cppjit_debug_break_python" +DEBUG_MODULE = os.path.join("cppjit", "debugger", "__init__.py") +INTERNAL_MARK = "[internal] " +WRAPPER_PREFIX = "__jc_" +_RUNTIME = re.compile( + r"^(libcppjit|libclangCppInterOp|libpython|libc[.-]|ld-linux|libffi|libstdc\+\+|libgcc_s)" +) +_PACKAGE = os.sep + "cppjit" + os.sep + + +class InternalFramesParameter(gdb.Parameter): + """Whether backtraces show the internal frames of cppjit, CppInterOp and CPython.""" + + set_doc = "Show or hide the internal frames of cppjit, CppInterOp and CPython." + show_doc = "Show whether backtraces show internal frames." + + def __init__(self) -> None: + super().__init__( + "cppjit internal-frames", + gdb.COMMAND_STACK, + gdb.PARAM_ENUM, + ["hide", "show"], + ) + self.value = "hide" + + def get_show_string(self, value: str) -> str: + return "Internal cppjit frames: %s." % value + + +_parameter: InternalFramesParameter | None = None + + +def hiding() -> bool: + return _parameter is None or _parameter.value == "hide" + + +class PythonFrame(FrameDecorator): + """A Python frame, shown at the address of the eval loop that runs it.""" + + def __init__( + self, base: FrameDecorator, frame: cpython.PyFrame, last: bool + ) -> None: + super().__init__(base) + self._base = base + self._frame = frame + self._last = last + self._variables: tuple[list[values.Variable], list[values.Variable]] | None = ( + None + ) + + def function(self) -> str: + return "[py] " + self._frame.name + + def address(self) -> int: + # gdb's DAP server needs an address for each frame. + return self._base.inferior_frame().pc() + + def filename(self) -> str: + return inferior.python_file(self._frame.filename) + + def python_file(self) -> str: + return self._frame.filename + + def line(self) -> int: + return self._frame.line + + def _read_variables( + self, + ) -> tuple[list[values.Variable], list[values.Variable]]: + if self._variables is None: + self._variables = values.variables(self._frame) + return self._variables + + def frame_args(self) -> list[values.Variable]: + return self._read_variables()[0] + + def frame_locals(self) -> list[values.Variable]: + return self._read_variables()[1] + + def elided(self) -> Iterator[FrameDecorator] | None: + if not self._last or hiding(): + return None + return iter([self._base]) + + +class InternalFrame(FrameDecorator): + """The top frame, in internal code.""" + + def function(self) -> str: + name = super().function() + if isinstance(name, int): + name = gdb.format_address(name) + return INTERNAL_MARK + str(name) + + +def is_internal_frame(frame: gdb.Frame) -> bool: + """Whether FRAME, a gdb.Frame, runs internal code.""" + try: + name = frame.name() or "" + if name.startswith(WRAPPER_PREFIX) or name == PYTHON_BREAK: + return True + if frame.find_sal().symtab is None: + return True + objfile = gdb.current_progspace().objfile_for_address(frame.pc()) + except (gdb.error, AttributeError): + return False + return bool(objfile and _RUNTIME.match(os.path.basename(objfile.filename))) + + +def _is_internal(item: FrameDecorator) -> bool: + if isinstance(item, PythonFrame): + return _PACKAGE in item.python_file() + return is_internal_frame(item.inferior_frame()) + + +def _hide_internal(items: Iterable[FrameDecorator]) -> Iterator[FrameDecorator]: + first = True + for item in items: + if not _is_internal(item): + yield item + elif first: + yield InternalFrame(item) + first = False + + +def _is_eval_loop(frame: gdb.Frame) -> bool: + try: + return (frame.name() or "").startswith(_EVAL_LOOP) + except gdb.error: + return False + + +class PythonFrameFilter: + def __init__(self) -> None: + self.name = FILTER_NAME + self.priority = 100 + self.enabled = getattr(gdb.frame_filters.get(self.name), "enabled", True) + gdb.frame_filters[self.name] = self + + def filter(self, frames: Iterator[FrameDecorator]) -> Iterator[FrameDecorator]: + try: + segments = inferior.python_segments() + except (cpython.Unsupported, gdb.error): + segments = [] + if segments: + frames = _skip_python_break(_merge(frames, segments)) + return _hide_internal(frames) if hiding() else frames + + +def _merge( + frames: Iterable[FrameDecorator], segments: list[list[cpython.PyFrame]] +) -> Iterator[FrameDecorator]: + pending = iter(segments) + for item in frames: + if _is_eval_loop(item.inferior_frame()): + segment = next(pending, None) + if segment: + for i, frame in enumerate(segment): + yield PythonFrame(item, frame, i == len(segment) - 1) + continue + yield item + + +def _skip_python_break(items: Iterator[FrameDecorator]) -> Iterator[FrameDecorator]: + """At a Python line breakpoint, start the stack at the Python line.""" + first = next(items, None) + if first is None: + return + try: + at_break = first.inferior_frame().name() == PYTHON_BREAK + except gdb.error: + at_break = False + if not at_break: + yield first + yield from items + return + for item in items: + if isinstance(item, PythonFrame) and not item.python_file().endswith( + DEBUG_MODULE + ): + yield item + yield from items + return + + +def _parameter_prefix() -> None: + if hasattr(gdb, "ParameterPrefix"): + gdb.ParameterPrefix("cppjit", gdb.COMMAND_NONE) + return + for verb in ("set", "show"): + gdb.Command("%s cppjit" % verb, gdb.COMMAND_NONE, prefix=True) + + +def install() -> None: + global _parameter + _parameter_prefix() + _parameter = InternalFramesParameter() + PythonFrameFilter() diff --git a/python/cppjit-debugger/gdb/inferior.py b/python/cppjit-debugger/gdb/inferior.py new file mode 100644 index 00000000..b123f964 --- /dev/null +++ b/python/cppjit-debugger/gdb/inferior.py @@ -0,0 +1,128 @@ +"""Access to the debugged process: memory, CPython state, environment, files.""" + +from __future__ import annotations + +import json +import os + +import gdb + +from . import cpython + +_runtimes: dict[tuple[int, int, int], cpython.Runtime] = {} +_indexes: dict[str, tuple[int, dict[str, str]]] = {} + + +def read(address: int, size: int) -> bytes: + if size <= 0: + return b"" + return bytes(gdb.selected_inferior().read_memory(address, size)) + + +def _address_of(expression: str) -> int | None: + try: + return int(gdb.parse_and_eval("(unsigned long)(%s)" % expression)) + except gdb.error: + return None + + +def runtime() -> cpython.Runtime: + """The cpython.Runtime of the selected inferior; raises cpython.Unsupported without one.""" + address = _address_of("&_PyRuntime") + if not address: + raise cpython.Unsupported("no _PyRuntime symbol") + inferior = gdb.selected_inferior() + key = (inferior.num, inferior.pid, address) + if key not in _runtimes: + try: + _runtimes[key] = cpython.Runtime(read, address, _address_of("&PyCode_Type")) + except gdb.MemoryError as exc: + raise cpython.Unsupported(str(exc)) + return _runtimes[key] + + +def python_segments( + thread: gdb.InferiorThread | None = None, +) -> list[list[cpython.PyFrame]]: + """The Python frames of THREAD (default: the selected thread), split per eval-loop run.""" + thread = thread or gdb.selected_thread() + if thread is None: + return [] + rt = runtime() + tstate = rt.thread_state(thread.ptid[1]) + return rt.segments(tstate) if tstate else [] + + +def _c_string(address: int, limit: int = 65536) -> bytes: + out = b"" + while len(out) < limit: + chunk = read(address + len(out), 256) + end = chunk.find(b"\0") + if end >= 0: + return out + chunk[:end] + out += chunk + return out + + +def _start_environment() -> dict[str, str]: + pid = gdb.selected_inferior().pid + try: + with open("/proc/%d/environ" % pid, "rb") as f: + entries = f.read().split(b"\0") + except OSError: + return {} + env = {} + for entry in entries: + name, _, value = entry.decode("utf-8", "replace").partition("=") + if name: + env[name] = value + return env + + +def environment() -> dict[str, str]: + """The environment of the selected inferior: libc's environ, so changes the process made + after its start count, and core files work. Variables of the start environment that environ + does not have are added: environ can lack some of them.""" + env = _start_environment() + pointer = _address_of("(char **) environ") + if pointer: + try: + for i in range(4096): + entry = int.from_bytes(read(pointer + 8 * i, 8), "little") + if not entry: + break + name, _, value = ( + _c_string(entry).decode("utf-8", "replace").partition("=") + ) + env[name] = value + return env + except gdb.MemoryError: + pass + return env or dict(os.environ) + + +def source_dir() -> str | None: + """The directory that receives the JIT inputs, or None.""" + return environment().get("CPPINTEROP_JIT_SOURCE_DIR") or None + + +def python_file(filename: str) -> str: + """A file with the source of FILENAME, a code object's file name.""" + if os.path.isfile(filename): + return filename + directory = source_dir() + pid = gdb.selected_inferior().pid + if not directory or not pid: + return filename + index = os.path.join(directory, "python-sources-%d.json" % pid) + try: + stamp = os.stat(index).st_mtime_ns + except OSError: + return filename + if _indexes.get(index, (None,))[0] != stamp: + try: + with open(index) as f: + _indexes[index] = (stamp, json.load(f)) + except (OSError, ValueError): + return filename + return _indexes[index][1].get(filename, filename) diff --git a/python/cppjit-debugger/gdb/lines.py b/python/cppjit-debugger/gdb/lines.py new file mode 100644 index 00000000..cfbdf75a --- /dev/null +++ b/python/cppjit-debugger/gdb/lines.py @@ -0,0 +1,83 @@ +"""Line breakpoints in code that the JIT has not compiled yet. + +The JIT compiles an inline function, for example in a header, only when +the code first uses it. Until then the line of a FILE:LINE breakpoint has +no code, and gdb moves the breakpoint to the next line that has code, which +can be in a different function. This module disables such a location in +JIT code if the lines between have code. When the JIT compiles the +function, gdb sets the breakpoint again and finds the line itself. +""" + +from __future__ import annotations + +import re + +import gdb + +_SPEC = re.compile(r"^(?:-source\s+(.+?)\s+-line\s+(\d+)|(.+):(\d+))$") +_JIT_OBJFILE = " int | None: + found = _SPEC.match((breakpoint.location or "").strip()) + if not found: + return None + return int(found.group(2) or found.group(4)) + + +def _is_filler(text: str) -> bool: + text = text.strip() + return not text or text.startswith(("//", "/*", "*")) or not text.strip("{}();") + + +def _moved_over_code(path: str, requested: int, actual: int) -> bool: + """Whether lines REQUESTED to ACTUAL - 1 of PATH have code.""" + if actual < requested: + return True + try: + with open(path, errors="replace") as f: + lines = f.read().splitlines() + except OSError: + return False + return not all(_is_filler(text) for text in lines[requested - 1 : actual - 1]) + + +def _in_jit_code(location: gdb.BreakpointLocation) -> bool: + objfile = gdb.current_progspace().objfile_for_address(location.address) + return bool(objfile and (objfile.filename or "").startswith(_JIT_OBJFILE)) + + +def _check(breakpoint: gdb.Breakpoint) -> None: + if breakpoint.type != gdb.BP_BREAKPOINT or not breakpoint.is_valid(): + return + requested = _requested_line(breakpoint) + if requested is None: + return + for location in breakpoint.locations: + key = (breakpoint.number, location.address) + source = location.source + if source is None or not _in_jit_code(location): + continue + wrong = source[1] != requested and _moved_over_code( + location.fullname or source[0], requested, source[1] + ) + if wrong and location.enabled: + _disabled.add(key) + location.enabled = False + elif not wrong and key in _disabled: + _disabled.discard(key) + location.enabled = True + + +def _on_breakpoint(breakpoint: gdb.Breakpoint) -> None: + try: + _check(breakpoint) + except (gdb.error, AttributeError, RuntimeError): + pass # gdb before 13 has no breakpoint locations. + + +def install() -> None: + gdb.events.breakpoint_created.connect(_on_breakpoint) + gdb.events.breakpoint_modified.connect(_on_breakpoint) diff --git a/python/cppjit-debugger/gdb/values.py b/python/cppjit-debugger/gdb/values.py new file mode 100644 index 00000000..cdbcd285 --- /dev/null +++ b/python/cppjit-debugger/gdb/values.py @@ -0,0 +1,170 @@ +"""Python objects as gdb values, for the variables of Python frames. + +A Python object becomes an lvalue of a marker type at the object's address. +A pretty-printer shows it like its repr() and gives the items of lists, +tuples and dicts, and the attributes of other objects, as children. Thus +"bt full" and the Variables view of a DAP client can show them. + +A cppjit proxy of a C++ object becomes the C++ object itself, if the debug +information has its type. +""" + +from __future__ import annotations + +import re +from typing import Iterator + +import gdb + +from . import cpython, inferior + +_objects: set[int] = set() +_types: dict[str, gdb.Type | None] = {} + + +def _marker() -> gdb.Type: + return gdb.lookup_type("unsigned long").volatile() + + +def _lookup_cpp_type(name: str) -> gdb.Type | None: + try: + return gdb.lookup_type(name) + except gdb.error: + pass + if not name.endswith(">"): + return None + # gdb knows a template instance only by its full name, with the default arguments. + pattern = re.escape(name[:-1]) + "[,>]" + listing = gdb.execute("info types -q ^%s" % pattern, to_string=True) + for found in re.findall( + r"^\s*(?:\d+:\s*)?(?:struct |class |union )?(.+?);?$", listing, re.M + ): + if re.match(pattern, found): + try: + return gdb.lookup_type(found) + except gdb.error: + continue + return None + + +def _cpp_type(name: str) -> gdb.Type | None: + if name not in _types: + _types[name] = _lookup_cpp_type(name) + return _types[name] + + +def _pointer_to(val: gdb.Value, target: gdb.Type, depth: int = 8) -> gdb.Value | None: + """The first field of VAL, searched depth first through bases and members, that is a + TARGET*.""" + tp = val.type.strip_typedefs() + if tp.code == gdb.TYPE_CODE_PTR: + pointee = tp.target().strip_typedefs().unqualified() + return val if pointee == target else None + if tp.code != gdb.TYPE_CODE_STRUCT or not depth: + return None + for field in tp.fields(): + if not hasattr(field, "bitpos"): + continue # static + found = _pointer_to(val[field], target, depth - 1) + if found is not None: + return found + return None + + +def _cpp_value(address: int) -> gdb.Value | None: + try: + found = inferior.runtime().cpp_object(address) + except (cpython.Unsupported, gdb.error): + return None + if not found: + return None + pointer, name, smart = found + tp = _cpp_type(name) + if smart is None: + return gdb.Value(pointer).cast(tp.pointer()).dereference() if tp else None + smart_tp = _cpp_type(smart[1]) + if smart_tp is None: + return None + holder = gdb.Value(smart[0]).cast(smart_tp.pointer()).dereference() + # Show the object that the smart pointer holds, as Python does. + held = _pointer_to(holder, tp.strip_typedefs().unqualified()) if tp else None + try: + if held is not None and int(held): + return held.dereference() + except gdb.error: + pass + return holder + + +def value(address: int) -> gdb.Value: + """A gdb.Value for the Python object at ADDRESS.""" + cpp = _cpp_value(address) + if cpp is not None: + return cpp + _objects.add(address) + return gdb.Value(address).cast(_marker().pointer()).dereference() + + +class _Printer: + def __init__(self, address: int, children: list[tuple[str, int]] | None) -> None: + self._address = address + self._children = children + + def to_string(self) -> str: + try: + return inferior.runtime().summary(self._address) + except (cpython.Unsupported, gdb.error): + return "" % self._address + + +class _ParentPrinter(_Printer): + def children(self) -> Iterator[tuple[str, gdb.Value]]: + for name, address in self._children or (): + yield name, value(address) + + +def _lookup(val: gdb.Value) -> _Printer | None: + if val.address is None or str(val.type) != str(_marker()): + return None + address = int(val.address) + if address not in _objects: + return None + try: + children = inferior.runtime().children(address) + except (cpython.Unsupported, gdb.error): + children = None + return _ParentPrinter(address, children) if children else _Printer(address, None) + + +class Variable: + """A Python variable, in the SymValueWrapper form of frame decorators.""" + + def __init__(self, name: str, address: int) -> None: + self._name = name + self._address = address + + def symbol(self) -> str: + return self._name + + def value(self) -> gdb.Value: + return value(self._address) + + +def variables(frame: cpython.PyFrame) -> tuple[list[Variable], list[Variable]]: + """The arguments and the locals of FRAME, a cpython.PyFrame, as two lists of Variable.""" + try: + arguments, local = inferior.runtime().variables(frame) + except (cpython.Unsupported, gdb.error): + return [], [] + return [Variable(*v) for v in arguments], [Variable(*v) for v in local] + + +def install() -> None: + gdb.pretty_printers.append(_lookup) + gdb.events.cont.connect(_forget) + + +def _forget(event: gdb.ContinueEvent) -> None: + # The addresses are valid only until the process runs again, and new JIT code can add types. + _objects.clear() + _types.clear() diff --git a/python/cppjit-debugger/libcppjit.so-gdb.py b/python/cppjit-debugger/libcppjit.so-gdb.py new file mode 100644 index 00000000..9809da17 --- /dev/null +++ b/python/cppjit-debugger/libcppjit.so-gdb.py @@ -0,0 +1,15 @@ +"""gdb auto-loads this file with libcppjit.so; see debugger/gdb/cppjit_gdb.py.""" + +from __future__ import annotations + +import os + +_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "debugger", "gdb", "cppjit_gdb.py" +) +with open(_path) as _f: + # Not "source": a nested source keeps this file's __file__. + exec( + compile(_f.read(), _path, "exec"), + {"__file__": _path, "__name__": "cppjit_gdb_loader"}, + ) diff --git a/src/cpyrt/cpyrtModule.cxx b/src/cpyrt/cpyrtModule.cxx index 915ceb74..36990f41 100644 --- a/src/cpyrt/cpyrtModule.cxx +++ b/src/cpyrt/cpyrtModule.cxx @@ -1093,6 +1093,17 @@ extern "C" PyObject* PyInit_libcppjit() { // create the memory regulator static MemoryRegulator s_memory_regulator; +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT + // Experimental debugger support: cppjit.debugger prepares a debug session + // if CPPJIT_DEBUGGER_SUPPORT is set. A failure must not fail the import. + if (PyObject* debugger = PyImport_ImportModule("cppjit.debugger")) { + Py_XDECREF(PyObject_CallMethod(debugger, "_configure", nullptr)); + Py_DECREF(debugger); + } + if (PyErr_Occurred()) + PyErr_Print(); +#endif + Py_INCREF(gThisModule); return gThisModule; } diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index a4c6b537..6be86faf 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -39,6 +39,11 @@ static inline size_t CALL_NARGS(size_t nargs) { return nargs & ~DIRECT_CALL; } #include #include +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT +#include +#include +#endif + std::recursive_mutex InterOpMutex; // builtin types @@ -131,6 +136,27 @@ static bool loadDispatchAPI(const InterOpPaths& Paths) { return true; } +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT +// Experimental debugger support (CPPJIT_ENABLE_DEBUGGER_SUPPORT): +// CPPJIT_DEBUGGER_SUPPORT set to anything but "" or "0" starts a debug +// session. +static bool debugSession() { + const char* env = getenv("CPPJIT_DEBUGGER_SUPPORT"); + return env && *env && strcmp(env, "0") != 0; +} + +// The interpreter writes each input to a file in CPPINTEROP_JIT_SOURCE_DIR, +// for the debugger; cppjit.debugger uses the same default. +static void setDefaultSourceDir() { + if (getenv("CPPINTEROP_JIT_SOURCE_DIR")) + return; + const char* tmp = getenv("TMPDIR"); + std::string dir = tmp && *tmp ? tmp : "/tmp"; + dir += "/cppjit-debug-" + std::to_string(getuid()); + setenv("CPPINTEROP_JIT_SOURCE_DIR", dir.c_str(), 0); +} +#endif + // CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside // CreateInterpreter, so nothing needs to be forwarded from here. static interop::TInterp_t @@ -154,6 +180,15 @@ acquireOrCreateInterpreter(const InterOpPaths& Paths) { args.push_back("-resource-dir"); args.push_back(resourceDir.c_str()); } +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT + if (debugSession()) { + args.push_back("-g"); + args.push_back("-O0"); + setDefaultSourceDir(); + // Output of JIT code shows at once when a debugger stops the process. + setvbuf(stdout, nullptr, _IOLBF, BUFSIZ); + } +#endif return Cpp::CreateInterpreter(args, /*GpuArgs=*/{}); } @@ -169,6 +204,10 @@ static void configureInterpreter(const InterOpPaths& Paths) { // set opt level (default to 2 if not given; Cling itself defaults to 0) int optLevel = 2; +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT + if (debugSession()) + optLevel = 0; +#endif if (getenv("CPPJIT_OPT_LEVEL")) optLevel = atoi(getenv("CPPJIT_OPT_LEVEL")); @@ -265,6 +304,50 @@ extern "C" int LoadCppInterOp() { return Loaded; } +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT +// cppjit.debugger calls this at a Python line breakpoint; the gdb extension +// breaks here. The write keeps the call from being folded away. +// The gdb extension increments the generation at each stop, so that +// cppjit.debugger reads breakpoints set during the stop before the next line. +// It sets the step mode (1 into, 2 over, 3 out, 4 return) and the thread for +// a step through Python lines. +// During a step, each call into JIT code calls cppjit_debug_enter_jit with +// its Cpp::JitCall, where the gdb extension can find the called function, +// and cppjit_debug_leave_jit after it, which calls the hook of +// cppjit.debugger. +#if defined(__GNUC__) +#define CPPJIT_DEBUG_HOOK __attribute__((noinline)) +#else +#define CPPJIT_DEBUG_HOOK +#endif +extern "C" { +RPY_EXPORTED void cppjit_debug_break_python(); +RPY_EXPORTED void cppjit_debug_enter_jit(const void* call); +RPY_EXPORTED void cppjit_debug_leave_jit(); +RPY_EXPORTED volatile int cppjit_debug_python_generation = 0; +RPY_EXPORTED volatile int cppjit_debug_python_step = 0; +RPY_EXPORTED volatile unsigned long cppjit_debug_python_step_thread = 0; +RPY_EXPORTED void (*volatile cppjit_debug_python_return_hook)() = nullptr; +} + +extern "C" void cppjit_debug_break_python() { + static volatile int Hits = 0; + Hits = Hits + 1; +} + +// The debugger breaks in these hooks: they must not be inlined. +extern "C" CPPJIT_DEBUG_HOOK void cppjit_debug_enter_jit(const void* call) { + static const void* volatile Last = nullptr; + if (Last != call) + Last = call; +} + +extern "C" CPPJIT_DEBUG_HOOK void cppjit_debug_leave_jit() { + if (auto* Hook = cppjit_debug_python_return_hook) + Hook(); +} +#endif + // local helpers ------------------------------------------------------------- static inline char* cppstring_to_cstring(const std::string& cppstr) { char* cstr = (char*)malloc(cppstr.size() + 1); @@ -890,6 +973,11 @@ static inline bool WrapperCall(interop::TCppMethod_t method, size_t nargs, if (Cpp::JitCall JC = Cpp::MakeFunctionCallable(method)) { InterOpMutex.unlock(); bool runRelease = false; +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT + const bool stepping = cppjit_debug_python_step; + if (stepping) + cppjit_debug_enter_jit(&JC); +#endif // const auto& fgen = /* is_direct ? faceptr.fDirect : */ faceptr; if (nargs <= cpyrt::SMALL_ARGS_N) { void* smallbuf[cpyrt::SMALL_ARGS_N]; @@ -905,6 +993,10 @@ static inline bool WrapperCall(interop::TCppMethod_t method, size_t nargs, JC.Invoke(result, {buf.data(), nargs}, self); // _CLING_CATCH_UNCAUGHT } +#ifdef CPPJIT_WITH_DEBUGGER_SUPPORT + if (stepping || cppjit_debug_python_step) + cppjit_debug_leave_jit(); +#endif if (runRelease) release_args(args, nargs); return true; diff --git a/test/test_debugging.py b/test/test_debugging.py new file mode 100644 index 00000000..ff10db29 --- /dev/null +++ b/test/test_debugging.py @@ -0,0 +1,1093 @@ +"""Source-level debugging of JIT-compiled C++ with gdb (experimental). + +The gdb tests need a cppjit built with CPPJIT_ENABLE_DEBUGGER_SUPPORT; the +decoder tests read the gdb extension from the source tree. + +Each gdb test runs a Python script that does cppdef and then calls the C++ +function, under gdb in batch mode or under gdb's DAP server. +""" + +import glob +import importlib.util +import json +import os +import queue +import re +import shutil +import subprocess +import sys +import textwrap +import threading + +import pytest + +GDB = shutil.which("gdb") +# Settings of the inferior that can break gdb, which has its own libraries and Python. +_INFERIOR_ONLY = ("LD_LIBRARY_PATH", "LD_PRELOAD", "PYTHONHOME", "PYTHONPATH") + +SCRIPT = '''\ +import cppjit + +cppjit.cppdef(""" +int jit_scale(int value, int factor) { + int product = value * factor; + int result = product + 1; // RESULT + return result; +} +int jit_scale_twice(int value) { return jit_scale(jit_scale(value, 2), 3); } +namespace jit_ns { struct Point { int x = 1; double y = 2.5; }; } +""") +cppjit.cppdef(""" +#include +std::shared_ptr jit_shared() { return std::make_shared(); } +std::unique_ptr jit_unique() { return std::make_unique(); } +""") + + +def compute(value): + data = {"scale": [value, 2.5], "name": "jit"} # DATA + point = cppjit.gbl.jit_ns.Point() + numbers = cppjit.gbl.std.vector["int"]([4, 5]) + shared, unique = cppjit.gbl.jit_shared(), cppjit.gbl.jit_unique() + return cppjit.gbl.jit_scale_twice(data["scale"][0]) # CALL_SITE + + +print("result", compute(5)) # MAIN +print("done") # AFTER +''' + + +def _line_of(text, marker): + return next(i for i, line in enumerate(text.splitlines(), 1) if marker in line) + + +def _gdb_version(): + try: + out = subprocess.run( + [GDB, "--version"], + capture_output=True, + text=True, + timeout=30, + env=_gdb_env(), + ).stdout + except (OSError, subprocess.TimeoutExpired): + return None + found = re.search(r"(\d+)\.(\d+)", out.splitlines()[0] if out else "") + return (int(found.group(1)), int(found.group(2))) if found else None + + +def _ptrace_denied(): + try: + with open("/proc/sys/kernel/yama/ptrace_scope") as f: + if int(f.read()) >= 3: + return "ptrace is disabled (kernel.yama.ptrace_scope=3)" + except (OSError, ValueError): + pass + # Containers can deny ptrace through seccomp: try it. + probe = subprocess.run( + [GDB, "-nx", "-batch", "-ex", "run", "--args", "/bin/sh", "-c", "exit 0"], + capture_output=True, + text=True, + timeout=120, + env=_gdb_env(), + ) + if "exited normally" not in probe.stdout: + return ( + "gdb cannot run a process: %s" + % (probe.stdout + probe.stderr).strip()[-200:] + ) + return None + + +def _gdb_env(): + return {k: v for k, v in os.environ.items() if k not in _INFERIOR_ONLY} + + +def _skip_reason(min_version): + if not sys.platform.startswith("linux"): + return "gdb tests run on Linux only" + if GDB is None: + return "no gdb on PATH" + version = _gdb_version() + if version is None or version < min_version: + return "needs gdb >= %d.%d, found %s" % (min_version + (version,)) + return _ptrace_denied() + + +# gdb 15 runs the program in the launch request, before configurationDone. +_DAP_GDB = (16, 0) + + +def _require_gdb(min_version=(12, 0)): + reason = _skip_reason(min_version) + if reason: + pytest.skip(reason) + + +needs_debug_offsets = pytest.mark.skipif( + sys.version_info < (3, 13), + reason="mixed backtraces read CPython 3.13+ debug offsets", +) + + +def _package_dir(): + return importlib.util.find_spec("cppjit").submodule_search_locations[0] + + +def _debugger_dir(): + return os.path.join(_package_dir(), "debugger") + + +def _gdb_script(): + return os.path.join(_debugger_dir(), "gdb", "cppjit_gdb.py") + + +# The gdb extension in the source tree; the decoders need no build. +_SOURCE_GDB_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + os.pardir, + "python", + "cppjit-debugger", + "gdb", +) +# Wheel tests run a copy of test/ without the source tree. +needs_source_gdb_dir = pytest.mark.skipif( + not os.path.isdir(_SOURCE_GDB_DIR), reason="no python/cppjit-debugger next to test/" +) + + +@pytest.fixture +def session(tmp_path): + """A directory with the script, and the environment of a debug session.""" + if not os.path.isdir(_debugger_dir()): + pytest.skip("cppjit is built without CPPJIT_ENABLE_DEBUGGER_SUPPORT") + (tmp_path / "script.py").write_text(SCRIPT) + env = dict( + os.environ, + CPPJIT_DEBUGGER_SUPPORT="1", + CPPINTEROP_JIT_SOURCE_DIR=str(tmp_path / "jit"), + ) + return tmp_path, env + + +def _run_gdb(cwd, env, commands, args=("script.py",), load="source " + _gdb_script()): + cmd = [ + GDB, + "-nx", + "-batch", + "-iex", + "set debuginfod enabled off", + "-iex", + "set pagination off", + ] + for name in _INFERIOR_ONLY: + if name in env: + cmd += ["-iex", "set environment %s %s" % (name, env[name])] + cmd += ["-iex", load, "-ex", "set breakpoint pending on"] + for command in commands: + cmd += ["-ex", command] + cmd += ["-ex", "kill", "--args", sys.executable] + list(args) + gdb_env = {k: v for k, v in env.items() if k not in _INFERIOR_ONLY} + out = subprocess.run( + cmd, cwd=str(cwd), env=gdb_env, capture_output=True, text=True, timeout=600 + ) + return out.stdout + out.stderr + + +def _jit_input(directory, text): + """The JIT input file in DIRECTORY that contains TEXT.""" + for path in glob.glob(os.path.join(str(directory), "input_*.cpp")): + with open(path) as f: + if text in f.read(): + return path + raise AssertionError("no JIT input with %r in %s" % (text, directory)) + + +@needs_source_gdb_dir +def test_line_table_decoder(): + """cpython.code_line agrees with code.co_lines() for each code unit""" + + cpython = _load_cpython() + + def sample(a, b): + total = 0 + for i in range(a): + if i % 2: + total += b * i + return total + + codes = [ + sample.__code__, + test_line_table_decoder.__code__, + textwrap.dedent.__code__, + subprocess.run.__code__, + ] + for code in codes: + for start, end, line in code.co_lines(): + for offset in range(start, end, 2): + found = cpython.code_line( + code.co_linetable, code.co_firstlineno, offset // 2 + ) + assert found == line, (code.co_name, offset) + + +def _load_cpython(): + spec = importlib.util.spec_from_file_location( + "_cppjit_gdb_cpython", os.path.join(_SOURCE_GDB_DIR, "cpython.py") + ) + cpython = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cpython) + return cpython + + +class _Point: + def __init__(self): + self.x = 1 + self.label = "p" + + +@needs_debug_offsets +@needs_source_gdb_dir +@pytest.mark.skipif(not os.path.exists("/proc/self/mem"), reason="no /proc/self/mem") +def test_variable_decoder(): + """cpython.Runtime reads the variables of a live frame of this process like repr()""" + + import ctypes + + cpython = _load_cpython() + with open("/proc/self/mem", "rb", buffering=0) as memory: + + def read(address, size): + memory.seek(address) + return memory.read(size) + + runtime = cpython.Runtime( + read, + ctypes.addressof(ctypes.c_char.in_dll(ctypes.pythonapi, "_PyRuntime")), + ) + + def sample(a, *rest, **extra): + big = -12345678901234567890 + ratio = 2.5 + text = "h\u00e9llo" + items = [1, "a", None, True, [2]] + table = {"k": (1,), 2: b"x"} + point = _Point() + materialized = _Point() + materialized.__dict__ + materialized.more = 3 + + def inner(): + return a + + tstate = runtime.thread_state(threading.get_native_id()) + frame = next( + f + for segment in runtime.segments(tstate) + for f in segment + if f.name.endswith("sample") + ) + # The frame must be live while it is read. + arguments, local = runtime.variables(frame) + found = dict(arguments + local) + return ( + [name for name, _ in arguments], + {name: runtime.summary(obj) for name, obj in found.items()}, + {name: runtime.children(obj) for name, obj in found.items()}, + (big, ratio, text, items, table, point, materialized, inner), + ) + + arguments, shown, children, _ = sample(7, 8, x=9) + # 3.13 has only co_argcount; 3.14 marks *args and **kwargs as arguments too. + assert arguments == ( + ["a", "rest", "extra"] if sys.version_info >= (3, 14) else ["a"] + ) + assert shown["a"] == "7" + assert shown["rest"] == "(8,)" and shown["extra"] == "{'x': 9}" + assert shown["big"] == "-12345678901234567890" and shown["ratio"] == "2.5" + assert shown["text"] == repr("h\u00e9llo") + assert shown["items"] == "[1, 'a', None, True, [...]]" + assert shown["table"] == "{'k': (...), 2: b'x'}" + assert shown["inner"].startswith(""]) as f: + assert f.read() == code + + +def test_break_by_function(session): + """break by name, show the source line, inspect locals, step""" + + _require_gdb() + tmp_path, env = session + out = _run_gdb( + tmp_path, + env, + [ + "break jit_scale", + "run", + "info locals", + "print value", + "next", + "next", + "print product", + "list", + ], + ) + assert re.search( + r"Breakpoint 1, jit_scale \(value=5, factor=2\) at .*input_\w+\.cpp:3", out + ), out + assert "int product = value * factor;" in out + assert re.search(r"^product = ", out, re.M) + assert "$1 = 5" in out + assert "return result;" in out + assert "$2 = 10" in out + + +def test_break_by_file_line(session): + """a pending file:line breakpoint binds in JIT code: input paths are the same on each run""" + + _require_gdb() + tmp_path, env = session + first = subprocess.run( + [sys.executable, "script.py"], + cwd=str(tmp_path), + env=env, + capture_output=True, + timeout=600, + ) + assert first.returncode == 0, first.stderr + path = _jit_input(tmp_path / "jit", "int jit_scale(int value") + with open(path) as f: + line = _line_of(f.read(), "// RESULT") + location = "%s:%d" % (os.path.basename(path), line) + out = _run_gdb( + tmp_path, env, ["break " + location, "run", "print product", "frame"] + ) + assert "Breakpoint 1 (%s) pending." % location in out, out + assert re.search( + r"Breakpoint 1, jit_scale \(value=5, factor=2\) at .*%s" % re.escape(location), + out, + ), out + assert "int result = product + 1; // RESULT" in out + assert "$1 = 10" in out + + +@needs_debug_offsets +def test_mixed_backtrace(session): + """the backtrace puts the Python caller's file:line between the C++ frames""" + + _require_gdb() + tmp_path, env = session + out = _run_gdb( + tmp_path, + env, + [ + "set cppjit internal-frames show", + "break jit_scale", + "run", + "bt", + "cppjit py-bt", + "cppjit gil", + ], + ) + bt = out[out.index("#0 ") :] + order = [ + r"#0 +jit_scale \(value=5, factor=2\) at \S+input_\w+\.cpp:3", + r"in jit_scale_twice \(value=5\) at \S+input_\w+\.cpp:\d+", + r"in \[py\] compute \(value=5\) at \S+script\.py:%d" + % _line_of(SCRIPT, "CALL_SITE"), + r"in \[py\] \(\) at \S+script\.py:%d" % _line_of(SCRIPT, "MAIN"), + r"Py_RunMain \(", + ] + position = 0 + for pattern in order: + found = re.compile(pattern).search(bt, position) + assert found, "%s not after offset %d in:\n%s" % (pattern, position, bt) + position = found.end() + assert 'return cppjit.gbl.jit_scale_twice(data["scale"][0]) # CALL_SITE' in out + assert re.search(r"Thread 1 \(LWP \d+\) holds the GIL\.", out), out + + +@needs_debug_offsets +def test_python_source_from_memory(session): + """the Python frames of `python -c` show a written copy of the code""" + + _require_gdb() + tmp_path, env = session + code = SCRIPT.replace("# CALL_SITE", "# IN_MEMORY") + out = _run_gdb( + tmp_path, + env, + ["break jit_scale", "run", "bt", "cppjit py-bt"], + args=("-c", code), + ) + found = re.search( + r"in \[py\] compute \(value=5\) at (\S+py_\w+string\.py):%d" + % _line_of(code, "IN_MEMORY"), + out, + ) + assert found, out + assert 'return cppjit.gbl.jit_scale_twice(data["scale"][0]) # IN_MEMORY' in out + + +@needs_debug_offsets +def test_auto_load(session): + """gdb loads the extension with libcppjit.so from a safe path""" + + _require_gdb() + tmp_path, env = session + out = _run_gdb( + tmp_path, + env, + ["break jit_scale", "run", "cppjit py-bt"], + load="add-auto-load-safe-path " + _package_dir(), + ) + assert "Traceback (most recent call last):" in out, out + assert 'return cppjit.gbl.jit_scale_twice(data["scale"][0]) # CALL_SITE' in out + + +NODE = shutil.which("node") +# A stop that does not come fails the test, not hangs it. +DAP_TIMEOUT = 300 +DAP_PROXY = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + os.pardir, + "examples", + "vscode-gdb", + "extension", + "dap-proxy.js", +) + + +class _Dap: + """A minimal client of gdb's DAP server over stdio.""" + + def __init__(self, cwd, transcript, prefix=()): + self._proc = subprocess.Popen( + list(prefix) + + [ + GDB, + "-nx", + "-i", + "dap", + "-iex", + "set debuginfod enabled off", + "-iex", + "source " + _gdb_script(), + ], + cwd=str(cwd), + env=_gdb_env(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + self._seq = 0 + self._pending = [] + self.transcript = transcript + self._messages = queue.Queue() + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): + while True: + length = None + while True: + header = self._proc.stdout.readline() + if not header: + self._messages.put(None) + return + header = header.decode().strip() + if not header: + break + if header.lower().startswith("content-length:"): + length = int(header.split(":", 1)[1]) + self._messages.put(json.loads(self._proc.stdout.read(length))) + + def _read(self): + try: + message = self._messages.get(timeout=DAP_TIMEOUT) + except queue.Empty: + raise TimeoutError("no DAP message in %d s" % DAP_TIMEOUT) from None + if message is None: + raise EOFError("gdb closed its DAP stream") + self.transcript.append(("<-", message)) + return message + + def send(self, command, **arguments): + self._seq += 1 + message = { + "seq": self._seq, + "type": "request", + "command": command, + "arguments": arguments, + } + self.transcript.append(("->", message)) + data = json.dumps(message).encode() + self._proc.stdin.write(b"Content-Length: %d\r\n\r\n" % len(data) + data) + self._proc.stdin.flush() + return self._seq + + def _next(self, wanted): + for i, message in enumerate(self._pending): + if wanted(message): + return self._pending.pop(i) + while True: + message = self._read() + if wanted(message): + return message + self._pending.append(message) + + def response(self, seq): + message = self._next( + lambda m: m.get("type") == "response" and m.get("request_seq") == seq + ) + assert message["success"], "%s failed: %s %s" % ( + message.get("command"), + message.get("message"), + message.get("body"), + ) + return message.get("body", {}) + + def request(self, command, **arguments): + return self.response(self.send(command, **arguments)) + + def event(self, name): + return self._next(lambda m: m.get("type") == "event" and m.get("event") == name) + + def close(self): + try: + self._proc.stdin.close() + self._proc.wait(timeout=60) + except (OSError, subprocess.TimeoutExpired): + self._proc.kill() + self._proc.wait() + + +@needs_debug_offsets +def test_dap_session(session): + """the VS Code path: gdb's DAP server binds a breakpoint in a JIT source file and shows a mixed stack""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + first = subprocess.run( + [sys.executable, "script.py"], + cwd=str(tmp_path), + env=env, + capture_output=True, + timeout=600, + ) + assert first.returncode == 0, first.stderr + path = _jit_input(tmp_path / "jit", "int jit_scale(int value") + with open(path) as f: + line = _line_of(f.read(), "// RESULT") + + transcript = [] + dap = _Dap(tmp_path, transcript) + try: + dap.request( + "initialize", + clientID="pytest", + adapterID="cppjit-gdb", + linesStartAt1=True, + columnsStartAt1=True, + ) + launch = dap.send( + "launch", + program=sys.executable, + args=["script.py"], + cwd=str(tmp_path), + env=env, + ) + dap.event("initialized") + breakpoints = dap.request( + "setBreakpoints", source={"path": path}, breakpoints=[{"line": line}] + ) + assert len(breakpoints["breakpoints"]) == 1 + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + assert stopped["reason"] == "breakpoint", stopped + frames = dap.request( + "stackTrace", threadId=stopped["threadId"], startFrame=0, levels=40 + )["stackFrames"] + top = frames[0] + assert top["name"].startswith("jit_scale") and top["line"] == line, top + assert os.path.samefile(top["source"]["path"], path), top + callers = [f for f in frames if f["name"] == "[py] compute"] + assert callers and callers[0]["line"] == _line_of(SCRIPT, "CALL_SITE"), frames + assert os.path.samefile( + callers[0]["source"]["path"], str(tmp_path / "script.py") + ) + scopes = dap.request("scopes", frameId=top["id"])["scopes"] + values = {} + for scope in scopes: + if scope["variablesReference"] and scope["name"] != "Registers": + for variable in dap.request( + "variables", variablesReference=scope["variablesReference"] + )["variables"]: + values[variable["name"]] = variable["value"] + assert values.get("value") == "5" and values.get("product") == "10", values + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close() + target = os.environ.get("CPPJIT_TEST_DAP_TRANSCRIPT") + if target: + with open(target, "w") as f: + for direction, message in transcript: + if ( + message.get("command") == "launch" + and message["type"] == "request" + ): + # Keep the user's environment out of the file. + arguments = dict( + message["arguments"], + env={k: env[k] for k in env if k.startswith("CPP")}, + ) + message = dict(message, arguments=arguments) + f.write("%s %s\n" % (direction, json.dumps(message))) + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_python_breakpoint(session): + """the example's DAP proxy stops at a Python line and shows it as the top frame""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + script = str(tmp_path / "script.py") + line = _line_of(SCRIPT, "CALL_SITE") + dap = _Dap(tmp_path, [], prefix=(NODE, DAP_PROXY)) + try: + dap.request("initialize", clientID="pytest", adapterID="cppjit-gdb") + launch = dap.send( + "launch", + program=sys.executable, + args=["script.py"], + cwd=str(tmp_path), + env=env, + ) + dap.event("initialized") + answer = dap.request( + "setBreakpoints", source={"path": script}, breakpoints=[{"line": line}] + ) + assert answer["breakpoints"][0]["verified"], answer + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + frames = dap.request( + "stackTrace", threadId=stopped["threadId"], startFrame=0, levels=40 + )["stackFrames"] + top = frames[0] + assert top["name"] == "[py] compute" and top["line"] == line, frames[:3] + assert os.path.samefile(top["source"]["path"], script), top + dap.request("continue", threadId=stopped["threadId"]) + assert dap.event("exited")["body"]["exitCode"] == 0 + dap.request("disconnect") + finally: + dap.close() + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_python_breakpoint_added_while_stopped(session): + """a Python breakpoint set at a C++ stop is hit right after Continue""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + script = str(tmp_path / "script.py") + dap = _Dap(tmp_path, [], prefix=(NODE, DAP_PROXY)) + try: + dap.request("initialize", clientID="pytest", adapterID="cppjit-gdb") + launch = dap.send( + "launch", + program=sys.executable, + args=["script.py"], + cwd=str(tmp_path), + env=env, + ) + dap.event("initialized") + dap.request("setFunctionBreakpoints", breakpoints=[{"name": "jit_scale"}]) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + dap.request( + "setBreakpoints", + source={"path": script}, + breakpoints=[{"line": _line_of(SCRIPT, "AFTER")}], + ) + dap.request("setFunctionBreakpoints", breakpoints=[]) + dap.request("continue", threadId=stopped["threadId"]) + stopped = dap.event("stopped")["body"] + top = dap.request( + "stackTrace", threadId=stopped["threadId"], startFrame=0, levels=1 + )["stackFrames"][0] + assert top["name"] == "[py] ", top + assert top["line"] == _line_of(SCRIPT, "AFTER"), top + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close() + + +def _proxy_launch(tmp_path, env): + """A proxy session that launched script.py; returns it and the launch request.""" + dap = _Dap(tmp_path, [], prefix=(NODE, DAP_PROXY)) + dap.request( + "initialize", + clientID="pytest", + adapterID="cppjit-gdb", + supportsVariableType=True, + ) + launch = dap.send( + "launch", program=sys.executable, args=["script.py"], cwd=str(tmp_path), env=env + ) + dap.event("initialized") + return dap, launch + + +def _top(dap, stopped): + return dap.request( + "stackTrace", threadId=stopped["threadId"], startFrame=0, levels=40 + )["stackFrames"] + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_python_step_over(session): + """Step Over at a return goes back to the caller's line, in the middle of that line""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + dap, launch = _proxy_launch(tmp_path, env) + try: + dap.request( + "setBreakpoints", + source={"path": str(tmp_path / "script.py")}, + breakpoints=[{"line": _line_of(SCRIPT, "CALL_SITE")}], + ) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + assert _top(dap, stopped)[0]["name"] == "[py] compute" + dap.request("next", threadId=stopped["threadId"]) + stopped = dap.event("stopped")["body"] + assert stopped["reason"] == "step", stopped + top = _top(dap, stopped)[0] + assert top["name"] == "[py] ", top + assert top["line"] == _line_of(SCRIPT, "MAIN"), top + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close() + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_python_step_out(session): + """Step Over goes to the next line of the function; Step Out stops in the caller's line""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + dap, launch = _proxy_launch(tmp_path, env) + try: + dap.request( + "setBreakpoints", + source={"path": str(tmp_path / "script.py")}, + breakpoints=[{"line": _line_of(SCRIPT, "DATA")}], + ) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + dap.request("next", threadId=stopped["threadId"]) + stopped = dap.event("stopped")["body"] + top = _top(dap, stopped)[0] + assert top["name"] == "[py] compute", top + assert top["line"] == _line_of(SCRIPT, "DATA") + 1, top + dap.request("stepOut", threadId=stopped["threadId"]) + stopped = dap.event("stopped")["body"] + assert stopped["reason"] == "step", stopped + top = _top(dap, stopped)[0] + assert top["name"] == "[py] ", top + assert top["line"] == _line_of(SCRIPT, "MAIN"), top + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close() + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_step_out_of_jit_code(session): + """Step Out of the outermost JIT function stops at the Python line that called it, not in glue code""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + dap, launch = _proxy_launch(tmp_path, env) + try: + dap.request( + "setFunctionBreakpoints", + breakpoints=[{"name": "jit_scale_twice"}], + ) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + frames = _top(dap, stopped) + assert frames[0]["name"].startswith("jit_scale_twice"), frames[0] + assert [f["name"] for f in frames[1:3]] == ["[py] compute", "[py] "], ( + frames + ) + dap.request("stepOut", threadId=stopped["threadId"]) + stopped = dap.event("stopped")["body"] + assert stopped["reason"] == "step", stopped + top = _top(dap, stopped)[0] + assert top["name"] == "[py] compute", top + assert top["line"] == _line_of(SCRIPT, "CALL_SITE"), top + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close() + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_python_step_into_cpp(session): + """Step Into at a Python line that calls C++ stops in the C++ function""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + dap, launch = _proxy_launch(tmp_path, env) + try: + dap.request( + "setBreakpoints", + source={"path": str(tmp_path / "script.py")}, + breakpoints=[{"line": _line_of(SCRIPT, "CALL_SITE")}], + ) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + dap.request("stepIn", threadId=stopped["threadId"]) + stopped = dap.event("stopped")["body"] + assert stopped["reason"] == "step", stopped + frames = _top(dap, stopped) + assert frames[0]["name"].startswith("jit_scale_twice"), frames[0] + assert frames[1]["name"] == "[py] compute", frames[1] + dap.request("continue", threadId=stopped["threadId"]) + assert dap.event("exited")["body"]["exitCode"] == 0 + dap.request("disconnect") + finally: + dap.close() + + +@needs_debug_offsets +def test_internal_frames(session): + """bt hides the cppjit and CPython frames unless internal-frames is show""" + + _require_gdb() + tmp_path, env = session + hidden = _run_gdb(tmp_path, env, ["break jit_scale", "run", "bt"]) + assert "[py] compute" in hidden, hidden + assert "CPPMethod::Execute" not in hidden and "__jc_" not in hidden, hidden + shown = _run_gdb( + tmp_path, + env, + ["set cppjit internal-frames show", "break jit_scale", "run", "bt"], + ) + assert "CPPMethod::Execute" in shown and "__jc_" in shown, shown + + +def test_up_skips_internal_frames(session): + """cppjit up goes from one JIT frame to its JIT caller and no further""" + + _require_gdb() + tmp_path, env = session + out = _run_gdb( + tmp_path, + env, + ["break jit_scale", "run", "cppjit up", "cppjit up", "cppjit down"], + ) + assert re.search(r"#\d+ .*jit_scale_twice", out), out + assert "No more frames outside cppjit internals." in out, out + assert re.search(r"#0 .*jit_scale \(", out), out + + +def _variables(dap, reference): + found = dap.request("variables", variablesReference=reference)["variables"] + return {v["name"]: v for v in found} + + +def _python_scopes(dap, frame): + scopes = dap.request("scopes", frameId=frame["id"])["scopes"] + return { + s["name"]: _variables(dap, s["variablesReference"]) + for s in scopes + if s["name"] != "Registers" + } + + +@needs_debug_offsets +@pytest.mark.skipif(NODE is None, reason="no node on PATH") +def test_dap_python_variables(session): + """a Python frame shows its arguments and locals, expands containers and answers hovers""" + + _require_gdb(_DAP_GDB) + tmp_path, env = session + dap, launch = _proxy_launch(tmp_path, env) + try: + dap.request( + "setBreakpoints", + source={"path": str(tmp_path / "script.py")}, + breakpoints=[{"line": _line_of(SCRIPT, "CALL_SITE")}], + ) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + top = _top(dap, stopped)[0] + scopes = _python_scopes(dap, top) + assert scopes["Arguments"]["value"]["value"] == "5", scopes + data = scopes["Locals"]["data"] + assert data["value"] == "{'scale': [...], 'name': 'jit'}", data + assert "type" not in data, data + items = _variables(dap, data["variablesReference"]) + assert items["'name'"]["value"] == "'jit'", items + scale = _variables(dap, items["'scale'"]["variablesReference"]) + assert [scale["0"]["value"], scale["1"]["value"]] == ["5", "2.5"], scale + hover = dap.request( + "evaluate", expression="data", frameId=top["id"], context="hover" + ) + assert hover["result"] == data["value"] and hover["variablesReference"], hover + point = scopes["Locals"]["point"] + fields = _variables(dap, point["variablesReference"]) + assert [fields["x"]["value"], fields["y"]["value"]] == ["1", "2.5"], fields + assert "vector", module + assert _python_scopes(dap, module)["Locals"]["compute"]["value"] == ( + "" + ) + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close() + + +@needs_debug_offsets +def test_python_variables_at_cpp_stop(session): + """at a C++ stop, bt full shows the variables of the Python callers""" + + _require_gdb() + tmp_path, env = session + out = _run_gdb(tmp_path, env, ["break jit_scale", "run", "bt full"]) + assert re.search(r"\[py\] compute \(value=5\)", out), out + assert "data = {'scale': [...], 'name': 'jit'}" in out, out + assert "compute = " in out, out + assert "point = {x = 1, y = 2.5}" in out, out + assert "shared = {x = 1, y = 2.5}" in out, out + assert "unique = {x = 1, y = 2.5}" in out, out + assert re.search(r"numbers = .*(_M_start|length 2)", out), out + + +HEADER = """\ +#pragma once +struct JitCounter { + void add(int x) { + int doubled = x * 2; + total += doubled; // ADD + } + // The constructor has code here, after ADD, before the JIT compiles add. + int total = 0; +}; +""" + +HEADER_SCRIPT = """\ +import os + +import cppjit + +cppjit.add_include_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "inc")) +cppjit.include("jit_counter.h") +counter = cppjit.gbl.JitCounter() +counter.add(4) +print("total", counter.total) +""" + + +def _header_session(session): + tmp_path, env = session + (tmp_path / "inc").mkdir() + header = tmp_path / "inc" / "jit_counter.h" + header.write_text(HEADER) + (tmp_path / "header.py").write_text(HEADER_SCRIPT) + return tmp_path, env, str(header) + + +@needs_debug_offsets +def test_break_in_included_header(session): + """a FILE:LINE breakpoint in a header waits for the JIT to compile its function""" + + _require_gdb((13, 0)) + tmp_path, env, header = _header_session(session) + line = _line_of(HEADER, "// ADD") + out = _run_gdb( + tmp_path, + env, + ["break jit_counter.h:%d" % line, "run", "bt 1", "print doubled", "next"], + args=("header.py",), + ) + assert re.search( + r"JitCounter::add \(this=\S+, x=4\) at \S*jit_counter\.h:%d" % line, out + ), out + assert "JitCounter::JitCounter" not in out, out # not moved to the constructor + assert "$1 = 8" in out, out + assert re.search(r"^%d\s+}" % (line + 1), out, re.M), out + + +@needs_debug_offsets +def test_dap_break_in_included_header(session): + """gdb's DAP server stops at a header line and gives the header's path""" + + _require_gdb(_DAP_GDB) + tmp_path, env, header = _header_session(session) + line = _line_of(HEADER, "// ADD") + dap = _Dap(tmp_path, []) + try: + dap.request("initialize", clientID="pytest", adapterID="cppjit-gdb") + launch = dap.send( + "launch", + program=sys.executable, + args=["header.py"], + cwd=str(tmp_path), + env=env, + ) + dap.event("initialized") + dap.request( + "setBreakpoints", source={"path": header}, breakpoints=[{"line": line}] + ) + dap.request("configurationDone") + dap.response(launch) + stopped = dap.event("stopped")["body"] + top = _top(dap, stopped)[0] + assert top["name"].startswith("JitCounter::add") and top["line"] == line, top + assert os.path.samefile(top["source"]["path"], header), top + dap.request("disconnect", terminateDebuggee=True) + finally: + dap.close()