Add experimental debugger support for JIT-compiled C++

.github/workflows/ci.yml

@@ -91,6 +91,57 @@ jobs:
           cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp"
           cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp"
 
+  # The experimental debugger support is off in the other cells, which skip the
+  # gdb tests of test_debugging.py.
+  debugger:
+    name: debugger/ubuntu-24.04/llvm22/py3.14
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.14'
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends cmake ninja-build g++ git libedit-dev
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+      # The DAP tests need gdb 16; Ubuntu 24.04 has gdb 15.
+      - uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-name: gdb-env
+          create-args: gdb=16.3
+          condarc: |
+            channels:
+              - conda-forge
+      - name: Add gdb to PATH
+        run: |
+          # Only gdb: the environment's python must not shadow setup-python's.
+          mkdir -p "$RUNNER_TEMP/gdb-bin"
+          ln -s "${MAMBA_ROOT_PREFIX:-$HOME/micromamba}/envs/gdb-env/bin/gdb" "$RUNNER_TEMP/gdb-bin/gdb"
+          echo "$RUNNER_TEMP/gdb-bin" >> "$GITHUB_PATH"
+      - uses: compiler-research/ci-workflows/actions/setup-llvm@main
+        with:
+          version: '22'
+          os: ubuntu-24.04
+      - name: Build and install CppJIT with the debugger support
+        run: |
+          CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" python -m pip install . -v \
+            --config-settings=cmake.define.LLVM_DIR="$LLVM_DIR" \
+            --config-settings=cmake.define.CPPJIT_ENABLE_DEBUGGER_SUPPORT=ON \
+            --config-settings=install.strip=false
+      - name: Run the debugging tests
+        working-directory: test
+        env:
+          CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++20
+        run: |
+          set -o pipefail
+          gdb --version | head -1
+          python -m pytest -ra --tb=short -p no:cacheprovider test_debugging.py | tee "$RUNNER_TEMP/debugging.log"
+          # A skip here means gdb or ptrace is not usable, so the cell tests nothing.
+          ! grep -q "^SKIPPED" "$RUNNER_TEMP/debugging.log"
+
   report:
     if: ${{ always() && github.event_name == 'pull_request' }}
     needs: build-test

.gitignore

@@ -30,4 +30,6 @@ venv/
 # Editors / OS
 .idea/
 .vscode/
+# the examples ship their launch configurations
+!/examples/*/.vscode/
 .DS_Store

CMakeLists.txt

@@ -10,8 +10,11 @@ include(GNUInstallDirs)
 # This option won't make a lot of sense since we only ship the shared library in site-packages
 # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests?
 option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF)
-set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "")
-set(CPPINTEROP_GIT_TAG "f4adaa6b50542dd564089fe584d407bf15351c97" CACHE STRING "")
+# Experimental: debugging of JIT-compiled C++ with gdb (cppjit.debugger, the
+# gdb extension and the libcppjit hooks). Off, the build has none of it.
+option(CPPJIT_ENABLE_DEBUGGER_SUPPORT "build the experimental debugger support" OFF)
+set(CPPINTEROP_GIT_REPOSITORY "https://github.com/chicagotrading/CppInterOp.git" CACHE STRING "")
+set(CPPINTEROP_GIT_TAG "upstreaming/jit-debugging" CACHE STRING "")
 set(CPPINTEROP_SOURCE_DIR "" CACHE PATH
     "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source")
 
@@ -77,6 +80,7 @@ target_compile_definitions(cppjit PRIVATE
     CPPJIT_CLANG_MAJOR="${CPPJIT_INTEROP_CLANG_MAJOR}"
     CPPJIT_CLANG_INCLUDE_DIR="${CPPJIT_INTEROP_CLANG_DIR}"
     $<$<BOOL:${CPPJIT_USE_CLING}>:CPPJIT_USE_CLING>
+    $<$<BOOL:${CPPJIT_ENABLE_DEBUGGER_SUPPORT}>:CPPJIT_WITH_DEBUGGER_SUPPORT>
 )
 
 target_include_directories(cppjit PRIVATE
@@ -116,10 +120,22 @@ if(NOT SKBUILD)
     set_target_properties(cppjit PROPERTIES
         LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/cppjit"
     )
+    set(_cppjit_debugger_tree)
+    if(CPPJIT_ENABLE_DEBUGGER_SUPPORT)
+        set(_cppjit_debugger_tree
+            COMMAND ${CMAKE_COMMAND} -E copy_directory
+                "${CMAKE_CURRENT_SOURCE_DIR}/python/cppjit-debugger"
+                "${CMAKE_BINARY_DIR}/python/cppjit/debugger"
+            COMMAND ${CMAKE_COMMAND} -E rename
+                "${CMAKE_BINARY_DIR}/python/cppjit/debugger/libcppjit.so-gdb.py"
+                "${CMAKE_BINARY_DIR}/python/cppjit/libcppjit.so-gdb.py"
+        )
+    endif()
     add_custom_target(cppjit-python-tree ALL
         COMMAND ${CMAKE_COMMAND} -E copy_directory
             "${CMAKE_CURRENT_SOURCE_DIR}/python/cppjit"
             "${CMAKE_BINARY_DIR}/python/cppjit"
+        ${_cppjit_debugger_tree}
     )
 endif()
 
@@ -137,6 +153,19 @@ if(NOT SKBUILD)
     )
 endif()
 
+# The experimental debugger support installs as cppjit.debugger, with the
+# gdb auto-load script beside the extension.
+if(CPPJIT_ENABLE_DEBUGGER_SUPPORT)
+    install(DIRECTORY python/cppjit-debugger/
+        DESTINATION cppjit/debugger
+        PATTERN "__pycache__" EXCLUDE
+        PATTERN "libcppjit.so-gdb.py" EXCLUDE
+    )
+    install(FILES python/cppjit-debugger/libcppjit.so-gdb.py
+        DESTINATION cppjit
+    )
+endif()
+
 # An external CppInterOp is consumed in place and nothing of it ships;
 # the bundled one installs with the builtin headers of the build clang,
 # laid out as a headers-only resource dir (only include/ ships).

README.md

@@ -121,6 +121,29 @@ reflection and JIT compilation API for cppjit.
 - Domain-specific languages: user-defined "pythonizations" adapt the
   bindings into Pythonic libraries.
 
+### Debugging JIT-compiled C++ (experimental)
+
+Build cppjit with `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON` to debug the
+JIT-compiled C++ at the source level with gdb, also from Python. The
+default build does not have this support. In a build with it, set
+`CPPJIT_DEBUGGER_SUPPORT=1` before `import cppjit`. cppjit then compiles
+with `-g -O0` and writes each C++ input to a file in
+`$TMPDIR/cppjit-debug-<uid>` (`CPPINTEROP_JIT_SOURCE_DIR` selects another
+directory), so that gdb shows the source and binds `file:line`
+breakpoints.
+
+The gdb extension in `cppjit/debugger/gdb` puts the Python frames between
+the C++ frames in backtraces, with their variables (CPython 3.13 or
+later), and adds the `cppjit` commands, such as `cppjit py-bt`:
+
+```bash
+CPPJIT_DEBUGGER_SUPPORT=1 gdb -iex "source $(python -c 'import cppjit.debugger as d; print(d.gdb_script())')" \
+    -ex 'break -qualified my_function' -ex run --args python my_script.py
+```
+
+For VS Code, with Python breakpoints and steps between Python and C++,
+see [examples/vscode-gdb](examples/vscode-gdb/README.md).
+
 ### Requirements
 
 A package manager installation of cppjit (such as pip) requires

docs/ReleaseNotes.md

@@ -16,7 +16,13 @@ This release supports Python 3.12-3.14 and LLVM 21-22.
 
 ## New features
 
-- ...
+- Experimental debugger support, in builds with
+  `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON` only. `CPPJIT_DEBUGGER_SUPPORT=1`
+  starts a debug session: the JIT compiles with `-g -O0`, and the C++
+  inputs go to files that gdb can show. A gdb extension
+  (`cppjit/debugger/gdb`) shows mixed Python and C++ backtraces with the
+  variables of both. `examples/vscode-gdb` adds Python breakpoints and
+  steps between Python and C++ in VS Code, through gdb's DAP server.
 
 ## Deprecated features
 

examples/vscode-gdb/.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "cppjit: demo.py (gdb)",
+            "type": "cppjit-gdb",
+            "request": "launch",
+            "program": "python3",
+            "args": ["${workspaceFolder}/demo.py"],
+            "cwd": "${workspaceFolder}",
+            "env": {"CPPJIT_DEBUGGER_SUPPORT": "1"}
+        },
+        {
+            "name": "cppjit: attach (gdb)",
+            "type": "cppjit-gdb",
+            "request": "attach",
+            "program": "python3",
+            "pid": "${input:pid}"
+        }
+    ],
+    "inputs": [
+        {"id": "pid", "type": "promptString", "description": "Process ID of the Python process"}
+    ]
+}

examples/vscode-gdb/README.md

@@ -0,0 +1,180 @@
+# Debug JIT-compiled C++ from VS Code
+
+`demo.py` compiles a C++ function with `cppjit.cppdef`, and a C++ class
+from the header `demo.h` with `cppjit.include`, and calls them from
+Python. With this example you can stop in the C++ source, step through it,
+see its variables, and see the Python lines that called it in the same
+call stack.
+
+The debugger support is experimental. It can change or go away, and it is
+not in the default build.
+
+## Requirements
+
+- A cppjit built with `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON`. Without it,
+  cppjit has no `cppjit.debugger` package, no gdb extension and no
+  debugger hooks.
+- Linux, and gdb 16 or later (for its DAP server). Tested with gdb 16.3.
+- CPython 3.13 or later for the Python frames in the call stack: the gdb
+  extension reads CPython's `_Py_DebugOffsets` table, so CPython does not
+  need debug information.
+- A CppInterOp that supports `CPPINTEROP_JIT_SOURCE_DIR`, for breakpoints
+  by file and line in `cppdef` code. Breakpoints by function name work
+  without it.
+- A ptrace scope that lets gdb debug its child processes
+  (`/proc/sys/kernel/yama/ptrace_scope` 0 or 1; attach needs 0).
+
+## Steps
+
+1. Install the example extension. It has no build step:
+
+   ```sh
+   ln -s "$PWD/extension" ~/.vscode/extensions/cppjit-gdb
+   ```
+
+   Then reload VS Code.
+
+2. Open this directory in VS Code. In `.vscode/launch.json`, set `program`
+   to a Python interpreter that can import cppjit.
+
+3. Set a breakpoint. Click in the gutter of `demo.py`, for example at the
+   `stats = ...` line in `report`. Or, in Run and Debug, add a function breakpoint
+   `summarize` (Breakpoints, `+`). Or click in the gutter of `demo.h`, for
+   example in `Histogram::add`. Or run the demo once, then open the
+   C++ input from the source directory and click in the gutter:
+
+   ```sh
+   grep -l summarize "${TMPDIR:-/tmp}/cppjit-debug-$(id -u)"/input_*.cpp
+   ```
+
+   The file name is a hash of the code, so the breakpoint stays valid for
+   the next runs until the code changes.
+
+4. Start "cppjit: demo.py (gdb)". VS Code stops at the breakpoint. The
+   call stack shows the C++ frames of `summarize`, then `[py] report` and
+   `[py] <module>`. Click a `[py]` frame to open `demo.py` at the line
+   that called the C++ code.
+
+`CPPJIT_DEBUGGER_SUPPORT=1` in the launch configuration makes cppjit
+compile with `-g -O0` and write each C++ input to a file.
+`CPPINTEROP_JIT_SOURCE_DIR` selects another directory. Python and C output are line buffered in a
+debug session, so the output shows before each stop.
+
+## Python breakpoints
+
+gdb cannot bind a breakpoint in a `.py` file, so `extension/dap-proxy.js`
+sits between VS Code and gdb. It answers the `setBreakpoints` requests for
+Python files itself and writes the lines to a JSON file. The launch request
+passes that file to the process in `CPPJIT_PY_BREAKPOINTS`. `cppjit.debugger`
+then uses `sys.monitoring` (Python 3.12 or later) to call the native
+function `cppjit_debug_break_python` before each of these lines, and the
+cppjit gdb extension stops there and shows the Python line as the top
+frame. The process reads the file again when it changes, so breakpoints
+can be added while it runs.
+
+The same works without VS Code: write the JSON file
+(`{"/path/demo.py": [28]}`), set `CPPJIT_PY_BREAKPOINTS` to its path, and
+run under gdb with the extension loaded.
+
+## Stepping and internal frames
+
+The call stack hides the internal frames by default: the cppjit and
+CppInterOp runtime, CPython, the C runtime, and the `__jc_*` call
+wrappers that CppInterOp generates. If a stop is in internal code, the
+top frame stays, marked `[internal]`. Set `"internalFrames": "show"` in
+the launch configuration to see and step through all frames.
+
+- At a Python line, Step Over, Step Into and Step Out step through Python
+  lines. The proxy turns them into `cppjit py-step` and a continue. Step
+  Over at a `return`, and Step Out, stop in the caller's line.
+- Step Into at a Python line that calls C++ stops at the first line of
+  the C++ function, if it has debug information (JIT code has it). For a
+  constructor, it stops in the constructor.
+- In C++ code, the steps are gdb's own. A step that leaves the outermost
+  C++ function stops at the Python line that called it, not in the
+  wrappers.
+
+In the gdb CLI, `cppjit py-step [into|over|out]` steps through Python
+lines and into C++, `cppjit py-step return` goes from C++ code to the
+Python line that called it, `cppjit up` and `cppjit down` skip internal frames, and
+`set cppjit internal-frames show` shows all frames in `bt`.
+
+## Breakpoints in headers
+
+The JIT compiles an inline function, such as `Histogram::add` in
+`demo.h`, only when the code first calls it. Until then, gdb finds no code
+at a line in it. gdb then moves a breakpoint to the next line with code,
+which can be in a different function, for example a constructor. The
+cppjit gdb extension disables such a location if the lines between have
+code. When the JIT compiles the function, gdb sets the breakpoint again at
+the correct line.
+
+## Python variables
+
+Each Python frame shows its arguments and its local variables. Module and
+class bodies show the names in their namespace, without the `__dunder__`
+names. The gdb extension reads the objects from the process memory, so
+this also works at a C++ stop and in a core file:
+
+- `int`, `float`, `bool`, `None`, `str` and `bytes` show like `repr()`.
+- Lists, tuples and dicts show a short preview, and expand to their items.
+- A C++ object that cppjit returns to Python shows as the C++ object,
+  as in a C++ frame: it expands to its fields. An object in a smart
+  pointer, such as `std::shared_ptr<T>`, shows as the `T` that it holds,
+  or as the smart pointer if it is null. gdb's pretty-printers
+  apply, for example to `std::vector`, if gdb loads them: gdb loads the
+  libstdc++ printers only from a directory in its auto-load safe path
+  (`add-auto-load-safe-path <libstdc++ directory>` in the gdbinit).
+- Other objects show `<Type object at 0x...>`, and expand to their
+  instance attributes.
+
+In a Python frame, the proxy answers a hover, a watch or a console
+expression that is a Python name, such as `point.x` or `items[0]`, from
+these variables. In the gdb CLI, `bt full` shows them.
+
+## Why gdb's DAP server
+
+The extension starts `gdb -i dap` with the cppjit gdb extension loaded.
+gdb's own DAP server uses gdb's frame filters for the call stack, so the
+Python frames appear in VS Code without more code, and the smoke test
+below can drive the same server. The Microsoft C/C++ extension talks to
+gdb through MI instead. It can also show JIT code, but frame filters need
+`-enable-frame-filters` there, and gdb 16 can hang in
+`-stack-list-arguments` when a frame filter meets an optimized-out
+artificial `this`.
+
+`test/test_debugging.py::test_dap_session` is that smoke test. It drives
+the same `gdb -i dap` command without VS Code: it sets a breakpoint in a
+JIT input file, and checks the stack and the variables.
+
+## Without VS Code
+
+```sh
+CPPJIT_DEBUGGER_SUPPORT=1 gdb -iex "source $(python -c 'import cppjit.debugger as d; print(d.gdb_script())')" \
+    -ex 'break -qualified summarize' -ex run --args python demo.py
+```
+
+`bt` shows the Python frames between the C++ frames.
+`cppjit sources [REGEX]` lists the JIT inputs, or the lines that match
+REGEX as `FILE:LINE`. `cppjit py-bt` prints the Python stack.
+`cppjit gil` shows the thread that holds the GIL. gdb also loads the
+extension by itself with `add-auto-load-safe-path <cppjit package
+directory>`.
+
+Use `break -qualified NAME` for a global function: `break NAME` also
+matches each class member with that name, for example in LLVM.
+
+## Limits
+
+- Python variables are read only. There is no evaluation of Python
+  expressions, only of names. Objects with `__slots__` show no
+  attributes. C++ types that the debug information does not have show as
+  Python objects.
+- Python breakpoints and Python steps need a launch, not an attach.
+- A breakpoint or a step on a line that already ran can need up to 0.2 s
+  of run time to take effect: CPython stops the line events of lines that
+  had no breakpoint, and a thread turns them on again.
+- In the CLI, a Python frame shows the address of the eval loop that runs
+  it, and frames without debug information show `at LIBRARY` in place of
+  `from LIBRARY` (gdb prints all frames that way when a frame filter is
+  active).