diff --git a/.coveragerc b/.coveragerc index e312baf..4aaecbe 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,8 +3,6 @@ branch = True data_file = build/coverage/.coverage source = netgraph_core -omit = - python/netgraph_core/_docs.py [report] show_missing = True diff --git a/.github/ci-constraints.txt b/.github/ci-constraints.txt new file mode 100644 index 0000000..d0991f1 --- /dev/null +++ b/.github/ci-constraints.txt @@ -0,0 +1,78 @@ +# Exact versions for CI's Python environments and for the coverage venv of `make cov`, so a new +# release of a tool cannot change a CI result without a change in this repository: the accepted +# development environment's versions, and for what only Linux, Windows or Python 3.11 installs, +# the version a resolver chose for those platforms (its marker says where it applies). pip itself +# is one of them: a job installs this pip (`pip install -U -c` this file, which also downgrades a +# newer one) before anything else. They reach a build environment only through the mechanism of +# the tool that fills it: pip's own isolated build takes --build-constraint or +# PIP_BUILD_CONSTRAINT (since pip 26.2 never -c or PIP_CONSTRAINT), `python -m build` takes +# PIP_CONSTRAINT, and its uv installer (the release wheels) UV_CONSTRAINT. NumPy 2.5 needs Python +# 3.12; Python 3.11 gets the last NumPy that supports it. Update deliberately. +build==1.5.0 +certifi==2026.7.22 +cfgv==3.5.0 +charset-normalizer==3.5.1 +cmake==4.4.2 +colorlog==6.12.0 +coverage==7.15.4 +distlib==0.4.3 +docutils==0.23 +filelock==3.32.3 +gcovr==8.6 +id==1.6.1 +identify==2.6.19 +idna==3.19 +iniconfig==2.3.0 +jaraco.classes==3.4.0 +jaraco.context==6.1.2 +jaraco.functools==4.6.0 +Jinja2==3.1.6 +keyring==25.7.0 +lxml==6.1.2 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +more-itertools==11.1.0 +nh3==0.3.6 +ninja==1.13.0 +nodeenv==1.10.0 +packaging==26.3 +pathspec==1.1.1 +pip==26.2.1 +platformdirs==4.11.3 +pluggy==1.6.0 +pre_commit==4.6.2 +pybind11==3.1.0 +pybind11-stubgen==3.0.0 +Pygments==2.21.0 +pyproject_hooks==1.2.0 +pyright==1.1.401 +pytest==9.1.1 +pytest-cov==7.1.0 +python-discovery==1.5.2 +PyYAML==6.0.3 +readme_renderer==45.0 +requests==2.34.2 +requests-toolbelt==1.0.0 +rfc3986==2.0.0 +rich==15.0.0 +ruff==0.11.13 +scikit_build_core==1.0.3 +setuptools==84.0.0 +twine==7.0.0 +typing_extensions==4.16.0 +urllib3==2.7.0 +virtualenv==21.7.4 +wheel==0.48.0 +numpy==2.5.2; python_version >= "3.12" +numpy==2.4.4; python_version < "3.12" +backports.tarfile==1.2.0; python_version < "3.12" +importlib_metadata==9.0.1; python_version < "3.12" +zipp==4.1.1; python_version < "3.12" +cffi==2.1.1; sys_platform == "linux" +cryptography==50.0.2; sys_platform == "linux" +jeepney==0.9.0; sys_platform == "linux" +pycparser==3.11; sys_platform == "linux" +SecretStorage==3.5.0; sys_platform == "linux" +colorama==0.4.6; sys_platform == "win32" +pywin32-ctypes==0.2.3; sys_platform == "win32" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 85d5023..04fe87b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,8 +48,11 @@ jobs: steps: - uses: actions/checkout@v4 - # Install uv for faster dependency installation (required on macOS, bundled in manylinux) + # Install uv for faster dependency installation (required on macOS, bundled in manylinux). + # An exact version, like the Python tools of .github/ci-constraints.txt. - uses: astral-sh/setup-uv@v4 + with: + version: "0.11.16" # Build wheels using cibuildwheel - name: Build wheels @@ -62,10 +65,26 @@ jobs: # Use uv for faster dependency installation (10-100x faster than pip) CIBW_BUILD_FRONTEND: "build[uv]" + # The pins of .github/ci-constraints.txt for what uv installs without constraints of + # cibuildwheel's own: the build requirements of every wheel (scikit-build-core and its + # dependencies; pybind11 is pinned in pyproject.toml) and the test requirements. uv + # reads UV_CONSTRAINT only when a call passes no -c, so cibuildwheel's own tools keep + # its pins. Linux builds in a container that holds the project at /project. + CIBW_ENVIRONMENT_LINUX: "UV_CONSTRAINT=/project/.github/ci-constraints.txt" + CIBW_ENVIRONMENT_MACOS: UV_CONSTRAINT="$GITHUB_WORKSPACE/.github/ci-constraints.txt" + CIBW_ENVIRONMENT_WINDOWS: UV_CONSTRAINT="$GITHUB_WORKSPACE/.github/ci-constraints.txt" # Target architecture per matrix entry (native compilation, no QEMU) CIBW_ARCHS: "${{ matrix.cibw_archs }}" - CIBW_TEST_COMMAND: 'python -c "import netgraph_core; print(netgraph_core.__version__)"' + # The Python suite on every built wheel (the shipped configuration, hooks off; the + # free-threaded cp314t wheels included): exactly the tests that need a hook skip. + # Windows runs pytest without dev/run-checks.sh, after the same check of the hook + # setting: the wheel lacks the private hooks. + CIBW_TEST_REQUIRES: "pytest==9.1.1" + CIBW_TEST_COMMAND: "PYTHON=python bash {project}/dev/run-checks.sh pytest off -p no:cacheprovider" + CIBW_TEST_COMMAND_WINDOWS: >- + python -c "import sys, _netgraph_core as m; sys.exit('the wheel carries the private hooks' if hasattr(m, '_max_flows_per_call') else 0)" + && python -m pytest {project}/tests/py -p no:cacheprovider CIBW_BEFORE_ALL_LINUX: "yum install -y libatomic || (apt-get update && apt-get install -y libatomic1) || apk add libatomic" - uses: actions/upload-artifact@v4 @@ -73,20 +92,59 @@ jobs: name: cibw-wheels-${{ matrix.os }}-${{ matrix.cibw_archs }} path: ./wheelhouse/*.whl + # build fills its isolated environment with a plain pip install, which PIP_CONSTRAINT reaches; + # the same variable pins pip itself and build. build_sdist: name: Build SDist runs-on: ubuntu-latest + env: + PIP_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - name: Build SDist - run: pipx run build --sdist + run: | + python -m pip install -U pip + python -m pip install build + python -m build --sdist - uses: actions/upload-artifact@v4 with: name: cibw-sdist path: dist/*.tar.gz + # An installation built from the source distribution, tested with the Python suite of the same + # commit. pip builds it in an isolated environment of its own, which since pip 26.2 takes only + # build constraints (PIP_BUILD_CONSTRAINT), never PIP_CONSTRAINT: both name the same file. + # PIP_CONSTRAINT pins the venv's pip itself and pytest. + test_sdist: + name: Test an installation built from the SDist + needs: [build_sdist] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/download-artifact@v4 + with: + name: cibw-sdist + path: sdist + - name: Build and install from the SDist + env: + PIP_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt + PIP_BUILD_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt + run: | + python -m venv "$RUNNER_TEMP/sdist-venv" + "$RUNNER_TEMP/sdist-venv/bin/python" -m pip install -U pip + "$RUNNER_TEMP/sdist-venv/bin/python" -m pip install pytest + "$RUNNER_TEMP/sdist-venv/bin/python" -m pip install -v sdist/*.tar.gz + - name: Python tests against the installation + run: PYTHON="$RUNNER_TEMP/sdist-venv/bin/python" bash dev/run-checks.sh pytest off -p no:cacheprovider + test_wheels_centos_stream_9: name: Test wheels (CentOS Stream 9, py${{ matrix.python-version }}) needs: [build_wheels] @@ -100,6 +158,8 @@ jobs: - name: Install Python run: dnf install -y python${{ matrix.python-version }} python${{ matrix.python-version }}-pip + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: pattern: cibw-wheels-ubuntu-* @@ -108,13 +168,16 @@ jobs: - name: Install and test wheel run: | - python${{ matrix.python-version }} -m pip install --upgrade pip - python${{ matrix.python-version }} -m pip install netgraph-core --find-links wheelhouse/ - python${{ matrix.python-version }} -c "import netgraph_core; print('CentOS Stream 9:', netgraph_core.__version__)" + python${{ matrix.python-version }} -m venv /tmp/wheel-venv + /tmp/wheel-venv/bin/python -m pip install --upgrade -c .github/ci-constraints.txt pip + /tmp/wheel-venv/bin/python -m pip install -c .github/ci-constraints.txt pytest numpy + # --no-index: the wheel built above, never a published one of the same version. + /tmp/wheel-venv/bin/python -m pip install --no-index --find-links wheelhouse/ netgraph-core + PYTHON=/tmp/wheel-venv/bin/python bash dev/run-checks.sh pytest off -p no:cacheprovider publish_pypi: name: Publish to PyPI - needs: [check_version, build_wheels, build_sdist, test_wheels_centos_stream_9] + needs: [check_version, build_wheels, build_sdist, test_sdist, test_wheels_centos_stream_9] runs-on: ubuntu-latest environment: pypi permissions: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba61abb..9af2762 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,10 @@ on: pull_request: workflow_dispatch: +# Every job installs its Python tools with the pins of .github/ci-constraints.txt, pip itself +# first. An editable install builds the package in an isolated environment of pip's own, which +# since pip 26.2 takes only build constraints (--build-constraint), never -c: both name the same +# file. jobs: test: name: Unit tests (${{ matrix.os }}, py${{ matrix.python-version }}) @@ -15,21 +19,41 @@ jobs: matrix: os: [ubuntu-22.04, macos-latest, windows-latest] python-version: ["3.11", "3.12", "3.13", "3.14"] + include: + # Free-threaded CPython (no GIL): the module declares py::mod_gil_not_used(). + - os: ubuntu-22.04 + python-version: "3.14t" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + # bash on every OS: PowerShell would not stop when the first command fails. - name: Install deps + shell: bash run: | - python -m pip install -U pip wheel - python -m pip install -e .[dev] - - name: Run CI checks (lint + C++/Python tests) + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt --build-constraint .github/ci-constraints.txt -e ".[dev]" + - name: Run CI checks (lint + C++ tests with and without the hooks + Python tests) if: runner.os != 'Windows' run: make check-ci + # Windows runs pytest without dev/run-checks.sh, so this step checks the hook setting that + # script would: the editable build carries the private hooks (no test then skips). A step of + # its own: PowerShell does not stop at a failing command inside one step. + - name: Check the editable build carries the private hooks (Windows) + if: runner.os == 'Windows' + run: python -c "import sys, _netgraph_core as m; sys.exit(0 if hasattr(m, '_max_flows_per_call') else 'the editable build lacks the private hooks')" - name: Run Python tests (Windows) if: runner.os == 'Windows' run: python -m pytest tests/py/ -v + # The timing tool of dev/perf compiles against the library (its adapter uses the public + # API and two private functions of the queue rule), and nothing else builds it: its own + # tests, which compile its driver with this checkout's library, run on one job, so a + # library change that breaks the tool fails here. macOS: the platform the tool is made + # and checked on (its build record and memory reading are macOS-first). + - name: Timing tool tests (macOS, py3.14) + if: runner.os == 'macOS' && matrix.python-version == '3.14' + run: make perf-tool-test test-centos-stream-9: name: CentOS Stream 9 (py${{ matrix.python-version }}) @@ -53,12 +77,71 @@ jobs: - name: Install Python deps run: | - python${{ matrix.python-version }} -m pip install -U pip wheel - python${{ matrix.python-version }} -m pip install -e .[dev] + python${{ matrix.python-version }} -m pip install -U -c .github/ci-constraints.txt pip + python${{ matrix.python-version }} -m pip install -c .github/ci-constraints.txt \ + --build-constraint .github/ci-constraints.txt -e ".[dev]" - - name: Run CI checks (lint + C++/Python tests) + - name: Run CI checks (lint + C++ tests with and without the hooks + Python tests) run: PYTHON=python${{ matrix.python-version }} make check-ci + # The C++ tests with MSVC: a Release build with CMake's link-time optimisation (the wheel's + # configuration), with and without the private hooks. + cpp-windows-msvc: + name: C++ tests (windows, MSVC) + runs-on: windows-latest + defaults: + run: + shell: bash + env: + CC: cl + CXX: cl + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build tools + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt cmake ninja + - uses: ilammy/msvc-dev-cmd@v1 + - name: C++ tests, private hooks on + run: bash dev/cpp-tests.sh release + - name: C++ tests, private hooks off + run: bash dev/cpp-tests.sh off + + sanitizers: + name: C++ tests (ubuntu, ASan + UBSan) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build tools + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt cmake ninja + - name: C++ tests under AddressSanitizer and UndefinedBehaviorSanitizer + run: bash dev/cpp-tests.sh sanitize + + # The generated type stub python/netgraph_core/__init__.pyi (make stubs-check): no drift from a + # fresh generation out of a wheel of this commit, the stub and py.typed in the wheel and the + # sdist, and the typed clients of tests/typing (valid: no diagnostic; invalid: exactly the + # expected ones) under pinned Pyright against the installed wheel and an installation built + # from the sdist. The script makes its own pinned venvs. Python 3.14: the version the + # committed stub is generated with. + stubs: + name: Type stub and typed clients (ubuntu, py3.14) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + - name: Stub drift, package contents, typed clients (wheel and sdist) + run: PYTHON=python bash dev/stubs/run.sh check + coverage: name: Coverage (ubuntu, py3.11) runs-on: ubuntu-22.04 @@ -67,10 +150,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install deps - run: | - python -m pip install -U pip wheel - python -m pip install -e .[dev] + # make cov builds its own pinned venv (build/coverage-venv) and fails on any failure. - name: Generate coverage (Python + C++) run: make cov - name: Upload coverage artifacts diff --git a/.gitignore b/.gitignore index eca4b33..af80017 100644 --- a/.gitignore +++ b/.gitignore @@ -1,229 +1,35 @@ -# Byte-compiled / optimized / DLL files +# Python __pycache__/ *.py[codz] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ *.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -.benchmarks/ -cover/ - -# Consolidated coverage artifacts -build/coverage/ - -# CTest output if created in project root -Testing/ -CTestTestfile.cmake - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock -#poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -#pdm.lock -#pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -#pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv -env/ venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -.vscode/ - -# Ruff stuff: +.venv/ +.pytest_cache/ .ruff_cache/ -# PyPI configuration file -.pypirc +# Build outputs: CMake and scikit-build directories, distributions, compiled extensions +build/ +dist/ +*.so -# Cursor -# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to -# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data -# refer to https://docs.cursor.com/context/ignore-files -.cursorignore -.cursorindexingignore +# Credentials of `make publish` +.pypirc -# OS-specific -.DS_Store +# The backup and reject files `patch` leaves beside the files it patches +*.orig +*.rej -# IDEs +# Editors and OS +.vscode/ .idea/ *.code-workspace +.DS_Store -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ -.pgo-profile/ +# Local output of browser-automation tools (Playwright's MCP server) +.playwright-mcp/ # Personal Superset lifecycle overrides .superset/config.local.json -# Performance harness run logs -dev/perf/**/*.log +# The performance tool's default output directory +/dev/perf/results/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dfd90b..3f8522a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,14 +4,12 @@ repos: hooks: - id: check-merge-conflict - id: check-yaml - # Recorded measurement artefacts and review patches must stay byte-exact. - id: end-of-file-fixer - exclude: ^dev/(research/|perf/.*/(results|review-[^/]+)/) - id: trailing-whitespace - exclude: ^dev/(research/|perf/.*/(results|review-[^/]+)/) - id: mixed-line-ending - id: debug-statements + # The revision matches the dev pin of Ruff (pyproject.toml). - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.13 hooks: @@ -19,9 +17,15 @@ repos: args: ["--fix"] - id: ruff-format - - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.401 + # Pyright needs the compiled extension _netgraph_core to resolve the package's imports, so it + # runs as make lint runs it: the pinned Pyright of the project's venv (make dev), whose + # interpreter has the extension built, over the whole project (pyproject.toml's settings). + # An isolated hook environment cannot hold the extension. + - repo: local hooks: - id: pyright - additional_dependencies: ["numpy>=1.22"] - # Pyright reads configuration from pyproject.toml + name: pyright + entry: venv/bin/python -m pyright --pythonpath venv/bin/python + language: system + types_or: [python, pyi] + pass_filenames: false diff --git a/AGENTS.md b/AGENTS.md index 917fa5f..cdda238 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,13 +3,20 @@ - Setup: `bash .superset/workspace.sh setup`. Each worktree gets its own venv; setup does not replace the Git hooks shared by worktrees. - Superset **Run**: `bash .superset/workspace.sh check` rebuilds the Python - extension, then runs lint, types, C++ and Python tests. Editable Python imports - alone do not rebuild changed C++. -- For C++ changes, also run `make sanitize-test` (ASan/UBSan; errors must fail). + extension, then runs lint, types, the C++ tests with and without the private + hooks, and the Python tests. Editable Python imports alone do not rebuild + changed C++. +- For C++ changes, also run `make sanitize-test` (ASan/UBSan) and + `make tsan-test` (ThreadSanitizer); errors must fail. - For downstream changes, run NetGraph's `dev/check_core_integration.sh` with this worktree's absolute path. Test the chosen build, not the published wheel. Confirm defects with a reproducer. Max-flow tests need feasibility, conservation -and optimality/cut checks with nonuniform costs, masks and cost-factor cases. -Keep bindings and `_docs.py` consistent. Performance claims need correctness -checks and A/B/A on one quiet machine. Local checks do not replace CI. +and optimality/cut checks with nonuniform costs, excluded nodes and edges, and +cost ceilings (Absolute, Stretch). +A binding signature or docstring change needs the regenerated type stub +(`bash dev/stubs/run.sh generate`; `make stubs-check` checks drift, package +contents and typed clients). Performance claims need correctness checks and +A/B/A on one quiet machine (`dev/perf/README.md`; `make perf-tool-test` after +changing that tool; CI runs it on one macOS job, as a library change can break +it). Local checks do not replace CI. diff --git a/CHANGELOG.md b/CHANGELOG.md index e2b5c1a..402430d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,151 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +A rebuilt Core (breaking): one numeric domain (integer units), one flow core, one placement +engine. It replaces the 0.11 API. + +### Changed + +- **Graph**: `Graph(num_nodes, src, dst, capacity_units, cost, resolution)` (replacing 0.11's + `StrictMultiDiGraph`) is the one constructor: capacities are `int64` units of an explicit + resolution (a power of two in `[2^-32, 2^32]`), costs are positive integers, and the public + `EdgeId` is the rank of an edge in `(src, cost, dst, input)` with `input_index()` mapping ids + to input positions. Validation reports the first defect in a fixed order (the resolution, the + unit values, the shapes, zero or negative costs, the remaining bounds including the cost-sum + guard), on the exact integers also beyond int64. Accessors return read-only NumPy views that + keep their graph alive, also when empty. + Construction numbers the edges by counting sort, in expected time linear in edges plus nodes + for typical costs; above 65,536 distinct costs, or when the hash table that ranks the costs + examines too many slots, it sorts all costs instead. Moving a graph no longer allocates. + `Eligibility` is built from excluded node ids and EdgeIds, with `Eligibility.intersect(a, b)`. + `to_units` (round half up) and `as_float` are the only conversions between physical values and + units. +- **Route structures**: `PredDAG.from_csr(graph, offsets, parents, via)` with the complete + validation (shape, ranges, endpoints, duplicates, cycles, ascending via within a node); a DAG + is stored over its member nodes and presents the CSR shape on demand. `Path(graph, hops)` + with `to_dag()`. `PathSelection`, `EdgeTieBreak` (`DETERMINISTIC`, `PREFER_HIGHER_PRIORITY`), + `EdgePriority` (`RESIDUAL`, `CAPACITY`, `HEADROOM`). +- **Ledger**: `FlowLedger(graph, residual=None)` (replacing 0.11's `FlowGraph`) is the integer + ledger: a baseline in units (validated within `[0, capacity_units]`), records as snapshots in + FlowId order (`records()`, `record(flow_id)` with `FlowId = (ledger uid, id)`, ids never + reused), `capacity_units()`, `residual()`, `allocated()`, `bottleneck(sources, elig)`, + `reset()` and `copy()` (both rejected while a policy is bound; `reset()` clears the records a + destroyed policy left behind). `FlowRecord` carries the flow's endpoints as sets, + placement, admission, bundle (`PredDAG`), route cost, `offered` (Lossy only), delivery and + per-edge allocations and drops; counters are 128-bit (exact Python ints). Policies place and + remove flows; `reset()` replaces an unbound ledger's baseline and clears its records. +- **Kernels**: `FlowPlacement` (`MAX_FLOW`, `WEIGHTED`, `EQUAL_BALANCED`) and `Admission` + (`LOSSLESS`, `LOSSY`). `MAX_FLOW` re-solves the flow with the flow core on its route over the free + capacity plus its own allocation; `WEIGHTED` splits in proportion to weights fixed at creation + from `priority` (highest averages, D'Hondt, EdgeId ties); `EQUAL_BALANCED` splits equally by + round robin from the lowest EdgeId; Lossy splits each offer per call and records transit drops. +- **Policies**: `FlowPolicy(ledger, sources, destinations, config, elig=None, cache=None)` + is the one placement engine. `FlowPolicyConfig(routing=..., placement=..., admission=..., + selection=...)` has four explicit axes; the routing is `CostOnly()`, `CapacityAware(...)` + (flow count, an optional maximum, an `Absolute` or `Stretch` cost ceiling, a minimum flow + bandwidth in units, reoptimisation, `DiminishingReturns(window, fraction)`), `Global()` or + `Pinned(bundles)`; values outside an enumeration raise `ValueError`. `place(volume, + outputs=None)` runs the per-source transaction (creation, reoptimisation, amounts, growth; at + most `MAX_FLOWS_PER_CALL` creations per call) or, on Global, the cumulative minimum-cost flow on + the residual plus the policy's own allocation, and returns `PlacementResult` (`placed`, + `remaining`, `termination`, `local_outcomes`, `routes_changed`, `weights_changed`, `drops`, + Global's `cost_distribution`, and `edge_flows`, `residual`, `bottleneck` and Global's + `min_cut` when requested through `OutputOptions`). A request of 0 is validated like any other. + `remove()`, `records()`, `record(flow_id)` (the policy's own records), `placed_demand()` and + `usable_bundles()` complete the policy. +- **Failure guarantee**: every public call either completes or raises and changes nothing, also + when memory runs out, a worker thread fails or the Python result cannot be built. Two limits: + running out of memory inside pybind11's own object construction (outside the library's checked + result construction) may crash the process; a `DagCache` passed to the call: a building cache + keeps the complete entries and statistics of the lookups that succeeded before the failure, and + a frozen cache keeps its entries and changes none of its statistics, except one miss when the + call raised `DagCacheMiss`. +- **Search**: `spf(graph, roots, SpfOptions(direction, selection, targets=None, residual=None, + min_residual=1, elig=None))` returns `SpfResult`. It searches from a root set (every root at + distance 0), `Direction.FORWARD` or `REVERSE`. Without targets it finds the distance of every reachable + node and returns `costs` (`UNREACHABLE` for the nodes not reached) and the full DAG of tight edges, + the edges on some shortest path. With targets it stops at the nearest targets' cost and + returns `target_cost` and the DAG of the walks between the roots and a nearest target (for a + reverse search, from the target to a root). With + `multipath=False` it returns the canonical single path: the walk of lowest cost, then (under + `PREFER_HIGHER_PRIORITY` only) of highest bottleneck, the smallest priority value along it, + then of fewest hops, then of lexicographically smallest edge-id sequence; with + `multi_edge=False` each node pair is first reduced to one parallel member. A + residual adds residual gating, which keeps only the edges with at least `min_residual` + residual units. Every search runs on one traversal core, with a bucket queue (nodes grouped by + distance) when the largest edge cost is at most 65,535 times the costs' greatest common + divisor, and a binary heap otherwise. +- **KSP and paths**: `ksp(graph, src, dst, KspOptions(k, selection, max_path_cost=None, + residual=None, elig=None))` returns the K shortest simple paths as `(cost, Path)` pairs. They + come in increasing (cost, -bottleneck under `PREFER_HIGHER_PRIORITY`, hops, edge ids) order, + truncated at `k` and at an inclusive `Absolute` or `Stretch` ceiling. The selection's + `multi_edge` lists parallel members as separate paths (`True`) or keeps one per node pair. + `resolve_to_paths(graph, dag, src, dst, split_parallel_edges=False, max_paths=None)` lists a + DAG's walks in (cost, hops, edge ids) order. +- **DagCache**: `DagCache(graph)` is keyed by the root set, the direction and the Eligibility. + Its values are the full static shortest-path DAGs (immutable, shared, valid after `clear()`), and + `stats()` reports `hits`, `misses`, `entries`, `bytes` and `live_values`. + - **Building mode:** one writer; a miss searches and inserts. + - **Frozen mode:** `freeze()` makes the cache read-only and shareable by every worker of an + evaluator. A frozen lookup miss raises `DagCacheMiss`, an exception the evaluators pass on + unchanged. `clear()` raises when frozen, and `thaw()` returns to building mode after the + caller has joined every reader. +- **Evaluators**: `capacity(graph, sources, destinations, CapacityOptions(...))` (with Global + and `MAX_FLOW` the exact maximum flow, its marginal cost curve and its minimum cut), + `batch_capacity(graph, pairs, opts, per_pair_elig=None, workers=None)` (pair order; the lowest + failing index raises) and `sensitivity(graph, sources, destinations, SensitivityOptions(...), + workers=None)` (every candidate edge and its signed unthresholded loss in units; a failed + deletion fails the call). A supplied building cache needs `workers == 1`; a frozen cache is + shared by every worker. With `workers=None` the count is 1 when a cache is supplied, otherwise + `NGRAPH_CORE_BATCH_THREADS` and `NGRAPH_CORE_SENSITIVITY_THREADS` (an integer in [1, 4096]; + 1 when unset or invalid). +- **Flow core**: one exact integer minimum-cost flow solver (successive shortest paths with + potentials, then Dinic phases for each cost level; each edge is its own arc, visited in + ascending EdgeId) serves Global routing, the `MAX_FLOW` kernel and the evaluators (sensitivity + repairs the baseline's solution for each removed edge instead of solving again); the private + entry `_netgraph_core._place_global_units` returns the value, the marginal cost curve, the edge + flows, the minimum cut, the potentials and six operation counters. The solver numbers its + network in `int32`, so a solve raises `ValueError`, before it sizes anything, on a network of + more than `INT32_MAX - 2` nodes or with more than `2^30 - 1` edges, sources and destinations + together; graphs of every accepted size still build and can be searched. +- **C++ headers** (under `netgraph/core/`): `types.hpp`, `graph.hpp` (`Graph`, `to_units`, + `as_float`; it includes `eligibility.hpp`), `eligibility.hpp`, `options.hpp` (the selection + axes, `FlowPlacement`, `Admission`), `pred_dag.hpp`, `flow_record.hpp` (`FlowId`, + `Counter128`, `FlowRecord`), `flow_ledger.hpp`, `flow_policy.hpp`, `dag_cache.hpp`, + `evaluators.hpp` and `search.hpp` (`spf`, `ksp`, `resolve_to_paths` and their options; the + queue choice `SpfQueue { Auto, Heap }` is a C++-only trailing parameter of `spf`); each + compiles alone and includes no `detail/` header. Python spells the + creation cap `MAX_FLOWS_PER_CALL` (C++ `kMaxFlowsPerCall`). +- **Typing**: the package ships `__init__.pyi`, a type stub generated from the compiled module + (pybind11-stubgen with reviewed overrides: exact integer and array types, NumPy result + dtypes, optional values, and no constructor for the result classes the module refuses to + construct), next to `py.typed`. The stub carries the binding docstrings. `bash dev/stubs/run.sh + generate` regenerates it; `make stubs-check` checks it against a fresh generation, the wheel + and sdist contents, and typed client programs. +- **Error messages**: the error for a finite request after `float('inf')` within an epoch no + longer cites a rule number. + +### Removed + +- The 0.11 library and its bindings: `FlowState`, the previous `FlowGraph` and `FlowPolicy` and + their configuration, the backend and algorithms facade (with its `Graph` handle), the search + entry points (`shortest_paths`, `shortest_paths_to`, `spf_dag`, `spf_to`, `k_shortest_paths` + with its `unique` flag, `PredDAG.resolve_to_paths`, `NGRAPH_CORE_SPF_QUEUE`), max-flow, + batch max-flow and the previous sensitivity, the SPF memo and `StateStamp`, `kMinFlow`, + `kEpsilon`, `constants.hpp`, and the profiling facility (the Python accessors, the C++ scopes + and `NGRAPH_CORE_PROFILE`). +- The hand-written typing module `netgraph_core/_docs.py`, replaced by the generated stub. +- The sensitivity entry status (`SensitivityStatus`, `SensitivityEntry.status` and `.reason`): + Successful entries were always `EVALUATED`; + `SensitivityEntry.loss` is now always an int, and a failed deletion fails the call. + +### Fixed + +- Fixed double frees and leaks when native allocation fails while constructing Python result + wrappers for `spf`, `ksp` and `resolve_to_paths`; these failures now raise `MemoryError`. + ## [0.11.0] - 2026-10-02 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f08d4c..1ce07b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,47 +1,34 @@ cmake_minimum_required(VERSION 3.23) project(netgraph_core LANGUAGES CXX) -# Silence pybind11 FindPython compatibility warning by selecting the modern mode -set(PYBIND11_FINDPYTHON NEW CACHE STRING "Use modern FindPython in pybind11") -# Optional C++ tests + +# ============================================================================= +# Options. Every build states them on its command line (pyproject.toml for pip builds, +# dev/cpp-tests.sh for the C++ test builds), so a value cached by an earlier configure of the same +# build directory never decides a later build. +# ============================================================================= + option(NETGRAPH_CORE_BUILD_TESTS "Build C++ unit tests" OFF) -if(NETGRAPH_CORE_BUILD_TESTS) - include(FetchContent) - FetchContent_Declare( - googletest - URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - ) - FetchContent_MakeAvailable(googletest) - enable_testing() -add_executable(netgraph_core_tests - tests/cpp/smoke_test.cpp - tests/cpp/flow_policy_tests.cpp - tests/cpp/strict_multidigraph_tests.cpp - tests/cpp/shortest_paths_tests.cpp - tests/cpp/flow_state_tests.cpp - tests/cpp/flow_graph_tests.cpp - tests/cpp/max_flow_tests.cpp - tests/cpp/k_shortest_paths_tests.cpp - tests/cpp/masking_tests.cpp -) - target_link_libraries(netgraph_core_tests PRIVATE netgraph_core GTest::gtest_main) - target_include_directories(netgraph_core_tests PRIVATE tests/cpp) - if(NETGRAPH_CORE_COVERAGE AND (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")) - # Atomic counters: see the NETGRAPH_CORE_COVERAGE block below. - target_compile_options(netgraph_core_tests PRIVATE -O0 -g --coverage) - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - target_compile_options(netgraph_core_tests PRIVATE -fprofile-update=atomic) - endif() - target_link_options(netgraph_core_tests PRIVATE --coverage) - endif() - include(GoogleTest) - gtest_discover_tests(netgraph_core_tests) +# The private hooks (test controls, and diagnostics such as the placement trace that the tests +# compare with the reference under tests/reference/policy/; neither is public API): +# ON for the editable development build (pyproject.toml) and the hooks-on test suites +# (dev/cpp-tests.sh), OFF for a wheel and for the hooks-off test suite. +option(NETGRAPH_CORE_PRIVATE_HOOKS "Build the private test hooks into the library and the extension" OFF) +option(NETGRAPH_CORE_COVERAGE "Enable C++ coverage instrumentation" OFF) +option(NETGRAPH_CORE_SANITIZE "Enable Address/Undefined sanitizers" OFF) +option(NETGRAPH_CORE_TSAN "Enable ThreadSanitizer" OFF) +option(NETGRAPH_CORE_NATIVE "Optimize for current CPU (not portable)" OFF) +if(NETGRAPH_CORE_SANITIZE AND NETGRAPH_CORE_TSAN) + # ThreadSanitizer cannot be combined with AddressSanitizer in one binary. + message(FATAL_ERROR "NETGRAPH_CORE_SANITIZE and NETGRAPH_CORE_TSAN are mutually exclusive") endif() +# Settings for every target: CMake applies them to targets created later, so they come first. set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +# Silence pybind11 FindPython compatibility warning by selecting the modern mode +set(PYBIND11_FINDPYTHON NEW CACHE STRING "Use modern FindPython in pybind11") # macOS: enforce consistent deployment target and architecture for wheels/builds if(APPLE) @@ -59,81 +46,195 @@ if(APPLE) endif() endif() -# Dependencies -find_package(pybind11 3 CONFIG QUIET) +# Dependencies. pybind11 is exactly the version pinned in pyproject.toml's build requirements, +# whose licence notice the wheel ships (third_party/pybind11/LICENSE). +find_package(pybind11 3.1.0 EXACT CONFIG QUIET) if(NOT pybind11_FOUND) - message(STATUS "pybind11 not found via config; using FetchContent") + message(STATUS "pybind11 3.1.0 not found via config; using FetchContent") include(FetchContent) FetchContent_Declare( pybind11 - GIT_REPOSITORY https://github.com/pybind/pybind11.git - GIT_TAG v3.0.0 + URL https://github.com/pybind/pybind11/archive/refs/tags/v3.1.0.tar.gz + URL_HASH SHA256=ef712655692a2e9bf7bb7874c022564a45f91d847ddee987e720cd9e28849665 DOWNLOAD_EXTRACT_TIMESTAMP TRUE ) FetchContent_MakeAvailable(pybind11) endif() +# ============================================================================= +# Targets +# ============================================================================= add_library(netgraph_core STATIC - src/strict_multidigraph.cpp - src/shortest_paths.cpp - src/k_shortest_paths.cpp - src/max_flow.cpp - src/flow_state.cpp - src/flow_graph.cpp + src/graph.cpp + src/eligibility.cpp + src/flow_core.cpp + src/pred_dag.cpp + src/search.cpp + src/kernels.cpp + src/flow_ledger.cpp + src/dag_cache.cpp src/flow_policy.cpp - src/cpu_backend.cpp - src/profiling.cpp + src/evaluators.cpp ) target_include_directories(netgraph_core PUBLIC $ $ ) -## C++ standard is set globally above. +# PUBLIC: the library, the extension and the tests see one value (0 or 1). +target_compile_definitions(netgraph_core PUBLIC NETGRAPH_CORE_PRIVATE_HOOKS=$) target_compile_features(netgraph_core PUBLIC cxx_std_20) -# Warnings -if(MSVC) - target_compile_options(netgraph_core PRIVATE /W4 /permissive-) -else() - target_compile_options(netgraph_core PRIVATE -Wall -Wextra -Wpedantic) -endif() - -# Python extension -pybind11_add_module(_netgraph_core bindings/python/module.cpp) +# Python extension: the module entry and its registration files (bindings/python/), in the +# order the entry calls them; the shared helpers are headers there. +pybind11_add_module(_netgraph_core + bindings/python/module.cpp + bindings/python/axes.cpp + bindings/python/units.cpp + bindings/python/eligibility.cpp + bindings/python/graph.cpp + bindings/python/recorder.cpp + bindings/python/pred_dag.cpp + bindings/python/path.cpp + bindings/python/flow_record.cpp + bindings/python/flow_ledger.cpp + bindings/python/routing.cpp + bindings/python/policy_config.cpp + bindings/python/placement_result.cpp + bindings/python/dag_cache.cpp + bindings/python/flow_policy.cpp + bindings/python/evaluators.cpp + bindings/python/paths.cpp + bindings/python/search.cpp + bindings/python/hooks.cpp +) +# The library's PUBLIC usage requirements (C++20, the hooks flag, the libc++ settings below) reach +# the extension through this link. target_link_libraries(_netgraph_core PRIVATE netgraph_core) -target_compile_features(_netgraph_core PRIVATE cxx_std_20) if(APPLE) # Prefer maximum runtime compatibility for libc++ on macOS target_compile_definitions(netgraph_core PUBLIC _LIBCPP_DISABLE_AVAILABILITY=1 _LIBCPP_ABI_VERSION=1) - target_compile_definitions(_netgraph_core PRIVATE _LIBCPP_DISABLE_AVAILABILITY=1 _LIBCPP_ABI_VERSION=1) endif() +# The targets that get the project's warnings and either the link-time optimisation (where the +# compiler supports it) or, in a coverage or sanitizer build, the instrumentation: the library, +# the extension and, in a test build, the two test executables (added below). The header +# self-containment check below is not one of them. The extra code-generation flags go to the +# library and the extension only. +set(NETGRAPH_CORE_PROJECT_TARGETS netgraph_core _netgraph_core) + +if(NETGRAPH_CORE_BUILD_TESTS) + include(FetchContent) + FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip + URL_HASH SHA256=1f357c27ca988c3f7c6b4bf68a9395005ac6761f034046e9dde0896e3aba00e4 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + ) + # MSVC: link googletest against the same (shared) C runtime as the project. + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + # googletest itself stays at C++17 (under C++20 its char8_t printer warns with Clang 21); the + # project's tests below are C++20 like the library. + set_target_properties(gtest gtest_main gmock gmock_main PROPERTIES CXX_STANDARD 17) + enable_testing() + add_executable(netgraph_core_tests + tests/cpp/graph_tests.cpp + tests/cpp/graph_layout_tests.cpp + tests/cpp/graph_concurrency_tests.cpp + tests/cpp/flow_core_tests.cpp + tests/cpp/flow_core_deletion_tests.cpp + tests/cpp/eligibility_tests.cpp + tests/cpp/pred_dag_tests.cpp + tests/cpp/search_tests.cpp + tests/cpp/ksp_tests.cpp + tests/cpp/dag_cache_tests.cpp + tests/cpp/kernels_tests.cpp + tests/cpp/cost_only_creation_tests.cpp + tests/cpp/flow_ledger_tests.cpp + tests/cpp/flow_policy_fixture_tests.cpp + tests/cpp/flow_policy_tests.cpp + ) + target_compile_definitions(netgraph_core_tests PRIVATE NETGRAPH_CORE_TEST_DATA="${CMAKE_CURRENT_SOURCE_DIR}/tests/data") + # Allocation-failure tests replace the global operator new: a separate executable. + add_executable(netgraph_core_fault_tests + tests/cpp/fault_injection.cpp + tests/cpp/flow_ledger_fault_tests.cpp + tests/cpp/flow_policy_fault_tests.cpp + tests/cpp/flow_core_deletion_fault_tests.cpp + tests/cpp/search_fault_tests.cpp + tests/cpp/graph_fault_tests.cpp + ) + foreach(tgt netgraph_core_tests netgraph_core_fault_tests) + target_link_libraries(${tgt} PRIVATE netgraph_core GTest::gtest_main) + target_include_directories(${tgt} PRIVATE tests/cpp) + endforeach() + # Every public header compiles on its own: one generated source file per header, holding only + # its include, built with the test executables. No public header names a private detail/ + # header in an #include (the configure stops on one). + file(GLOB NETGRAPH_CORE_PUBLIC_HEADERS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/include + CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/include/netgraph/core/*.hpp) + set(NETGRAPH_CORE_HEADER_CHECKS) + foreach(header ${NETGRAPH_CORE_PUBLIC_HEADERS}) + file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/include/${header} private_includes + REGEX "^[ \t]*#[ \t]*include[ \t]*[\"<][^\">]*detail/") + if(private_includes) + message(FATAL_ERROR "public header ${header} includes a private detail/ header: ${private_includes}") + endif() + string(MAKE_C_IDENTIFIER ${header} id) + file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/header_checks/${id}.cpp CONTENT "#include \"${header}\"\n") + list(APPEND NETGRAPH_CORE_HEADER_CHECKS ${CMAKE_CURRENT_BINARY_DIR}/header_checks/${id}.cpp) + endforeach() + add_library(netgraph_core_header_checks OBJECT ${NETGRAPH_CORE_HEADER_CHECKS}) + target_link_libraries(netgraph_core_header_checks PRIVATE netgraph_core) + add_dependencies(netgraph_core_tests netgraph_core_header_checks) + list(APPEND NETGRAPH_CORE_PROJECT_TARGETS netgraph_core_tests netgraph_core_fault_tests) + include(GoogleTest) + gtest_discover_tests(netgraph_core_tests) + # These sweeps make about 19,000 and 13,000 injection attempts, each with a clean retry; + # sensitivity also creates and joins worker threads. Debug sanitizer runs on small runners + # can exceed the runner's 120-second default. Keep every attempt and allow ten minutes only + # for these two tests; all other tests retain the runner's timeout. + set(NETGRAPH_CORE_LONG_FAULT_TESTS + "PolicyFaults.EvaluatorsLeaveNoPoisonedScratch:PolicyFaults.SensitivityWorkersFailWithoutTerminating") + gtest_discover_tests(netgraph_core_fault_tests TEST_FILTER "-${NETGRAPH_CORE_LONG_FAULT_TESTS}") + gtest_discover_tests(netgraph_core_fault_tests TEST_FILTER "${NETGRAPH_CORE_LONG_FAULT_TESTS}" + PROPERTIES TIMEOUT 600) +endif() + +# Warnings: the project's set on the library, the extension and the test executables (the list +# above); third-party code keeps its own. +if(MSVC) + set(NETGRAPH_CORE_WARNINGS /W4 /permissive-) +else() + set(NETGRAPH_CORE_WARNINGS -Wall -Wextra -Wpedantic) +endif() +foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) + target_compile_options(${tgt} PRIVATE ${NETGRAPH_CORE_WARNINGS}) +endforeach() + # ============================================================================= # Performance optimizations (disabled when coverage/sanitizers are enabled) # ============================================================================= -option(NETGRAPH_CORE_COVERAGE "Enable C++ coverage instrumentation" OFF) -option(NETGRAPH_CORE_SANITIZE "Enable Address/Undefined sanitizers" OFF) -option(NETGRAPH_CORE_NATIVE "Optimize for current CPU (not portable)" OFF) -option(NETGRAPH_CORE_FAST_MATH "Enable fast math (may affect precision)" OFF) -option(NETGRAPH_CORE_PGO_GENERATE "Instrument for PGO profile collection" OFF) -option(NETGRAPH_CORE_PGO_USE "Use PGO profile data (set NETGRAPH_CORE_PGO_DIR)" OFF) -set(NETGRAPH_CORE_PGO_DIR "${CMAKE_BINARY_DIR}/pgo" CACHE PATH "Directory for PGO profile data") - -# Only apply optimizations when not in debug/instrumentation mode -if(NOT NETGRAPH_CORE_COVERAGE AND NOT NETGRAPH_CORE_SANITIZE) - # LTO (Link Time Optimization) - enables cross-module inlining +# Apply the extra optimization settings when coverage and sanitizers are off (whatever the build +# type: a Debug build gets them too). +if(NOT NETGRAPH_CORE_COVERAGE AND NOT NETGRAPH_CORE_SANITIZE AND NOT NETGRAPH_CORE_TSAN) + # LTO (Link Time Optimization) - enables cross-module inlining. LANGUAGES CXX: googletest + # enables C as well, and the check would otherwise require LTO support for C too and turn LTO + # off in every test build. The test executables are LTO-linked like the extension, so the C++ + # tests exercise the code generation that ships. include(CheckIPOSupported) - check_ipo_supported(RESULT ipo_supported OUTPUT ipo_error) + check_ipo_supported(RESULT ipo_supported OUTPUT ipo_error LANGUAGES CXX) if(ipo_supported) message(STATUS "Enabling LTO (Link Time Optimization)") - set_target_properties(netgraph_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) - set_target_properties(_netgraph_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) + set_target_properties(${NETGRAPH_CORE_PROJECT_TARGETS} PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) else() message(STATUS "LTO not supported: ${ipo_error}") endif() + # MSVC gets no extra flags: its build type's optimisation (/O2 in Release) and CMake's LTO (/GL + # with the matching /LTCG link option) suffice. if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") # GCC-only: allow aggressive inlining by promising symbols won't be interposed if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -155,78 +256,33 @@ if(NOT NETGRAPH_CORE_COVERAGE AND NOT NETGRAPH_CORE_SANITIZE) target_compile_options(netgraph_core PRIVATE -march=native) target_compile_options(_netgraph_core PRIVATE -march=native) endif() - - # Fast math - trades IEEE compliance for speed - if(NETGRAPH_CORE_FAST_MATH) - message(STATUS "Enabling fast math optimizations") - target_compile_options(netgraph_core PRIVATE -ffast-math) - target_compile_options(_netgraph_core PRIVATE -ffast-math) - endif() - - # PGO (Profile-Guided Optimization) - uses runtime profile to guide inlining decisions - # GCC: -fprofile-generate/-fprofile-use, Clang: -fprofile-instr-* - if(NETGRAPH_CORE_PGO_GENERATE) - message(STATUS "PGO: Instrumenting for profile generation -> ${NETGRAPH_CORE_PGO_DIR}") - file(MAKE_DIRECTORY ${NETGRAPH_CORE_PGO_DIR}) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - target_compile_options(_netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - target_link_options(netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - target_link_options(_netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - else() - # Clang/AppleClang: use LLVM instrumentation PGO - target_compile_options(netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - target_compile_options(_netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - target_link_options(netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - target_link_options(_netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - endif() - elseif(NETGRAPH_CORE_PGO_USE) - message(STATUS "PGO: Using profile data from ${NETGRAPH_CORE_PGO_DIR}") - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(netgraph_core PRIVATE -fprofile-use=${NETGRAPH_CORE_PGO_DIR} -fprofile-correction) - target_compile_options(_netgraph_core PRIVATE -fprofile-use=${NETGRAPH_CORE_PGO_DIR} -fprofile-correction) - else() - # Clang/AppleClang: use merged .profdata file - target_compile_options(netgraph_core PRIVATE -fprofile-instr-use=${NETGRAPH_CORE_PGO_DIR}/default.profdata) - target_compile_options(_netgraph_core PRIVATE -fprofile-instr-use=${NETGRAPH_CORE_PGO_DIR}/default.profdata) - endif() - endif() - elseif(MSVC) - # /O2: full optimization, /GL: whole program optimization (MSVC's LTO) - target_compile_options(netgraph_core PRIVATE /O2 /GL) - target_compile_options(_netgraph_core PRIVATE /O2 /GL) - if(NETGRAPH_CORE_FAST_MATH) - target_compile_options(netgraph_core PRIVATE /fp:fast) - target_compile_options(_netgraph_core PRIVATE /fp:fast) - endif() endif() endif() # ============================================================================= -# Debug/instrumentation options (mutually exclusive with optimizations above) +# Coverage and sanitizer options (mutually exclusive with the extra optimizations above). A request +# the compiler cannot honour stops the configure: a build that ran its tests without the requested +# checker would otherwise report success. # ============================================================================= if(NETGRAPH_CORE_COVERAGE) message(STATUS "Enabling coverage instrumentation (disables optimizations)") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - # batch_max_flow and sensitivity_analysis run worker threads, and the default - # non-atomic profile counters corrupt under concurrent updates (gcov then - # emits impossible values such as "branch 4 taken -1"). - set(COV_FLAGS -O0 -g --coverage) - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - list(APPEND COV_FLAGS -fprofile-update=atomic) - endif() - foreach(tgt netgraph_core _netgraph_core) + # Atomic counters (GCC and Clang): the evaluators' worker threads corrupt the default + # non-atomic profile counters under concurrent updates (gcov then emits impossible values + # such as "branch 4 taken -1"). + set(COV_FLAGS -O0 -g --coverage -fprofile-update=atomic) + foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) target_compile_options(${tgt} PRIVATE ${COV_FLAGS}) target_link_options(${tgt} PRIVATE --coverage) endforeach() else() - message(WARNING "NETGRAPH_CORE_COVERAGE requires GCC or Clang") + message(FATAL_ERROR "NETGRAPH_CORE_COVERAGE requires GCC or Clang") endif() endif() if(NETGRAPH_CORE_SANITIZE) - message(STATUS "Enabling sanitizers (disables optimizations)") + message(STATUS "Enabling sanitizers (without the extra optimizations)") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") # Must be a CMake list: a single quoted string reaches the compiler as one # argument and clang rejects "-fsanitize=address,undefined -fno-omit-frame-pointer". @@ -237,18 +293,34 @@ if(NETGRAPH_CORE_SANITIZE) # and then silently ignored, which is indistinguishable from not checking at all. set(SAN_FLAGS -fsanitize=address,undefined -fno-omit-frame-pointer -fno-sanitize-recover=undefined) - set(SAN_TARGETS netgraph_core _netgraph_core) - if(TARGET netgraph_core_tests) - # The test executable links the sanitized static library, so it must be - # built and linked with the same flags to pull in the sanitizer runtime. - list(APPEND SAN_TARGETS netgraph_core_tests) - endif() - foreach(tgt ${SAN_TARGETS}) + # The test executables link the sanitized static library, so they are built and linked with + # the same flags to pull in the sanitizer runtime. + foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) target_compile_options(${tgt} PRIVATE ${SAN_FLAGS}) target_link_options(${tgt} PRIVATE ${SAN_FLAGS}) endforeach() else() - message(WARNING "NETGRAPH_CORE_SANITIZE requires GCC or Clang") + message(FATAL_ERROR "NETGRAPH_CORE_SANITIZE requires GCC or Clang") + endif() +endif() + +if(NETGRAPH_CORE_TSAN) + message(STATUS "Enabling ThreadSanitizer (without the extra optimizations)") + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + # The data-race check of the evaluators' worker threads and the shared DagCache. A CMake + # list, as SAN_FLAGS above. A report makes the process exit non-zero (TSan's exit code 66, + # or an abort under the halt_on_error=1 of dev/cpp-tests.sh), so ctest fails on any race. -O1 + # as the TSan documentation advises (the Debug build type adds -g); the instrumented + # suites at -O0 would be several times slower for no extra coverage. + set(TSAN_FLAGS -fsanitize=thread -fno-omit-frame-pointer -O1) + # The test executables link the instrumented static library: same compile and link flags, so + # they carry the TSan runtime and their own accesses are instrumented too. + foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) + target_compile_options(${tgt} PRIVATE ${TSAN_FLAGS}) + target_link_options(${tgt} PRIVATE ${TSAN_FLAGS}) + endforeach() + else() + message(FATAL_ERROR "NETGRAPH_CORE_TSAN requires GCC or Clang") endif() endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6cd3577..79fed30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ NetGraph-Core is a hybrid C++/Python project. You will need: - Python 3.11+ - C++20 compatible compiler (GCC 10+, Clang 12+, MSVC 2019+) - CMake 3.23+ -- Ninja (recommended) +- Ninja (the C++ test runner uses it; `make dev` installs CMake and Ninja into the venv) ### 1. Clone the repository @@ -37,7 +37,11 @@ This will: ### Running Tests -Run all checks (linting + C++ tests + Python tests): +Run all checks: rebuild the editable extension, then lint (ruff, pyright), the check that the +policy fixture data still equals its frozen reference (`tests/reference/policy/README.md`), the +C++ tests with and without the private test hooks, and the Python tests. It only verifies and +never edits a file; `make check-ci` is the same without the rebuild (CI installs the package +itself): ```bash make check @@ -46,8 +50,12 @@ make check Run specific test suites: ```bash -make cpp-test # C++ tests (GoogleTest) -make test # Python tests (pytest) +make cpp-test # C++ tests (GoogleTest), private hooks on +make cpp-test-off # C++ tests without the hooks (the shipped configuration) +make test # Python tests (pytest) +make sanitize-test # C++ tests under ASan/UBSan (required for C++ changes) +make tsan-test # C++ tests under ThreadSanitizer (required for C++ changes) +make cov # Coverage of the C++ and Python suites (its own venv and builds) ``` ### Code Style @@ -55,26 +63,43 @@ make test # Python tests (pytest) - **Python**: We use `ruff` for linting and formatting, and `pyright` for static type checking. - **C++**: We follow standard C++20 practices. -Auto-fix Python formatting: +Apply fixes (these edit files): ```bash -make format +make format # ruff formatting +make hooks # every pre-commit hook, including ruff --fix and whitespace fixers ``` +The commit hook and `make hooks` run Pyright as `make lint` does, from the `make dev` venv: it +needs the built extension to resolve the package's imports. + +### Type Stub + +`python/netgraph_core/__init__.pyi` is generated from the compiled module. After changing a +binding's signature or docstring, run `bash dev/stubs/run.sh generate` and commit the stub; +`make stubs-check` (also in CI) fails when the stub drifts or the typed client checks fail. + +### Behaviour Documentation + +[docs/semantics.md](docs/semantics.md) is the maintained statement of what the library computes. +A change to observable behaviour (a result, an order, an outcome, an error, a default) updates it +in the same change, and its examples must still print what the document shows: +`tests/py/test_docs_examples.py` runs them, and README's quick start, in the Python tests. + ## Release Process Releases are automated via GitHub Actions when a new tag is pushed. 1. Move the pending notes into a new `## [X.Y.Z] - YYYY-MM-DD` section in `CHANGELOG.md`, dated the day you tag. Mark anything that breaks consumers as **BREAKING**. -2. Bump `version` in `pyproject.toml` to the same `X.Y.Z`. CI does not cross-check the - tag against this value, so a mismatch publishes the wrong version silently. +2. Bump `version` in `pyproject.toml` to the same `X.Y.Z`. Release CI compares the tag + with this value and refuses to publish on a mismatch. 3. Commit and push. 4. Create and push a matching tag: ```bash - git tag v0.8.0 - git push origin v0.8.0 + git tag vX.Y.Z + git push origin vX.Y.Z ``` 5. The CI pipeline builds wheels and the sdist, tests them, and publishes to PyPI via diff --git a/Makefile b/Makefile index 3828d0f..e48550e 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ # NetGraph-Core Development Makefile -.PHONY: help venv clean-venv dev install install-native install-pgo check check-ci lint format test qt build clean check-dist publish-test publish info hooks check-python cpp-test cov sanitize-test rebuild +.PHONY: help venv clean-venv dev install install-native check check-ci lint format test build clean check-dist publish-test publish info hooks check-python cpp-test cpp-test-off cov sanitize-test tsan-test stubs-check perf-tool-test rebuild .DEFAULT_GOAL := help # -------------------------------------------------------------------------- # Python interpreter detection # -------------------------------------------------------------------------- -# VENV_BIN: path to local virtualenv bin directory -VENV_BIN := $(PWD)/venv/bin +# VENV_BIN: the checkout's virtualenv bin directory (CURDIR is the directory make runs in, -C included) +VENV_BIN := $(CURDIR)/venv/bin # PY_BEST: scan for newest supported Python (used when creating new venvs) # Supports 3.11-3.14 to match requires-python >=3.11 @@ -41,22 +41,27 @@ help: @echo "Setup & Installation:" @echo " make venv - Create a local virtualenv (./venv)" @echo " make dev - Full development environment (package + dev deps + hooks)" - @echo " make install - Install package (default optimizations: LTO, loop unrolling)" + @echo " make install - Install package (editable, private hooks; LTO, loop unrolling)" @echo " make install-native - Install with CPU-specific optimizations (faster, not portable)" - @echo " make install-pgo - Profile-guided optimization (two-phase build, experimental)" @echo " make clean-venv - Remove virtual environment" @echo " make rebuild - Clean and rebuild (respects CMAKE_ARGS)" @echo "" @echo "Code Quality & Testing:" - @echo " make check - Run pre-commit (auto-fix) + C++/Python tests, then lint" - @echo " make check-ci - Run non-mutating lint + tests (CI entrypoint)" + @echo " make check - Rebuild the editable extension, then everything check-ci runs" + @echo " make check-ci - Lint, fixture data check, C++ tests (hooks on and off), Python tests" + @echo " (the CI entrypoint)" + @echo " (check and check-ci only verify: they never edit a file)" @echo " make lint - Run only linting (non-mutating: ruff + pyright)" - @echo " make format - Auto-format code with ruff" + @echo " make format - Auto-format code with ruff (edits files)" @echo " make test - Run Python tests (pytest)" - @echo " make qt - Run quick tests only (exclude slow/benchmark)" - @echo " make cpp-test - Build and run C++ tests" - @echo " make cov - Coverage summary + XML + single-page combined HTML" + @echo " make cpp-test - Build and run C++ tests (private hooks on)" + @echo " make cpp-test-off - Build and run C++ tests without the hooks (the shipped configuration)" + @echo " make cov - Coverage (own venv and builds) + XML + single-page combined HTML" @echo " make sanitize-test - Build and run C++ tests with sanitizers" + @echo " make tsan-test - Build and run C++ tests with ThreadSanitizer" + @echo " make stubs-check - Type stub: drift, wheel/sdist contents, typed clients (own venvs)" + @echo " (regenerate the stub: bash dev/stubs/run.sh generate)" + @echo " make perf-tool-test - Tests of the A/B/A timing tool in dev/perf (after changing it; CI: macOS)" @echo "" @echo "Build & Package:" @echo " make build - Build distribution packages" @@ -67,7 +72,7 @@ help: @echo "" @echo "Utilities:" @echo " make info - Show project information" - @echo " make hooks - Run pre-commit on all files" + @echo " make hooks - Run pre-commit on all files (applies its fixes)" @echo " make check-python - Check if venv Python matches best available" # Allow callers to pass CMAKE_ARGS and MACOSX_DEPLOYMENT_TARGET consistently @@ -120,48 +125,33 @@ install: @echo "📦 Installing package (editable, default optimizations)" @$(DEV_ENV) $(PIP) install -e . +# The project options are stated in pyproject.toml for every build; a variant overrides one of them +# with -C (a config setting), so it cannot stay in the shared build directory's cache. install-native: @echo "📦 Installing with native CPU optimizations (-march=native)" @echo " Note: Binary will only work on this CPU architecture" - @$(ENV_MACOS) $(ENV_CC) $(ENV_CXX) CMAKE_ARGS="$(strip $(CMAKE_ARGS) -DNETGRAPH_CORE_NATIVE=ON)" $(PIP) install -e . - -# PGO profile stored outside build/ so it survives rebuild -PGO_DIR := $(PWD)/.pgo-profile - -install-pgo: - @echo "📦 PGO Build Phase 1/3: Instrumenting..." - @rm -rf build/ $(PGO_DIR) - @mkdir -p $(PGO_DIR) - @$(ENV_MACOS) $(ENV_CC) $(ENV_CXX) CMAKE_ARGS="$(strip $(CMAKE_ARGS) -DNETGRAPH_CORE_PGO_GENERATE=ON -DNETGRAPH_CORE_PGO_DIR=$(PGO_DIR) -DNETGRAPH_CORE_NATIVE=ON)" $(PIP) install -e . - @echo "📦 PGO Build Phase 2/3: Collecting profile (running benchmark)..." - @$(PYTHON) dev/benchmark_profiling_overhead.py --mesh-size 25 --spf-iters 1000 --flow-iters 200 >/dev/null 2>&1 - @# Clang: merge raw profiles into .profdata (profile may be in cwd or PGO_DIR) - @if ls $(PGO_DIR)/*.profraw >/dev/null 2>&1; then \ - echo " Merging Clang profile data..."; \ - xcrun llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw 2>/dev/null || \ - llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw; \ - elif ls *.profraw >/dev/null 2>&1; then \ - mv *.profraw $(PGO_DIR)/; \ - xcrun llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw 2>/dev/null || \ - llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw; \ - fi - @echo "📦 PGO Build Phase 3/3: Rebuilding with profile data..." - @rm -rf build/ - @$(ENV_MACOS) $(ENV_CC) $(ENV_CXX) CMAKE_ARGS="$(strip $(CMAKE_ARGS) -DNETGRAPH_CORE_PGO_USE=ON -DNETGRAPH_CORE_PGO_DIR=$(PGO_DIR) -DNETGRAPH_CORE_NATIVE=ON)" $(PIP) install -e . - @echo "✅ PGO build complete" + @$(DEV_ENV) $(PIP) install -e . -C cmake.define.NETGRAPH_CORE_NATIVE=ON +# Verification only: neither target edits a file (make format and make hooks apply fixes). check: - @PYTHON=$(PYTHON) bash dev/run-checks.sh + @$(MAKE) install + @$(MAKE) check-ci +# The fixture data check: tests/data/policy_fixtures.json still equals what its frozen reference +# computes (tests/reference/policy/README.md; standard library only; -B writes no __pycache__). check-ci: @$(MAKE) lint + @$(PYTHON) -I -B tests/reference/policy/export_policy_fixtures.py --check @$(MAKE) cpp-test - @$(MAKE) test + @$(MAKE) cpp-test-off + @PYTHON=$(PYTHON) bash dev/run-checks.sh pytest on +# Pyright resolves imports (NumPy in the type stub, the extension) through the interpreter it +# is given; without --pythonpath it would take whichever python3 comes first on PATH. lint: @$(RUFF) format --check . @$(RUFF) check . - @$(PYTHON) -m pyright + @$(PYTHON) -m pyright --pythonpath "$$($(PYTHON) -c 'import sys; print(sys.executable)')" format: @$(RUFF) format . @@ -169,9 +159,6 @@ format: test: @$(PYTEST) -qt: - @$(PYTEST) --no-cov -m "not slow and not benchmark" - build: @echo "🏗️ Building distribution packages..." @if $(PYTHON) -c "import build" >/dev/null 2>&1; then \ @@ -211,8 +198,7 @@ publish: clean: @echo "🧹 Cleaning build artifacts and cache files..." @rm -rf build/ dist/ *.egg-info/ || true - @rm -rf .pytest_cache .ruff_cache .mypy_cache htmlcov htmlcov-python .coverage coverage.xml coverage-*.xml .benchmarks .pytest-benchmark || true - @rm -rf Testing CTestTestfile.cmake || true + @rm -rf .pytest_cache .ruff_cache || true @find . -path "./venv" -prune -o -type f -name "*.pyc" -delete 2>/dev/null || true @find . -path "./venv" -prune -o -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true @find . -path "./venv" -prune -o -type f -name "*.pyo" -delete 2>/dev/null || true @@ -246,38 +232,26 @@ hooks: cpp-test: @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh release +cpp-test-off: + @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh off + +# The summary script's own rules first (its refusals make a failed measurement fail; no build). cov: - @echo "📦 Reinstalling with C++ coverage instrumentation..." - @$(PIP) install -U scikit-build-core "pybind11>=2.11" - @PIP_NO_BUILD_ISOLATION=1 CMAKE_ARGS="-DNETGRAPH_CORE_COVERAGE=ON" $(PIP) install -e .'[dev]' - @echo "🧪 Running Python tests with coverage..." - @mkdir -p build/coverage - @$(PYTEST) --cov=netgraph_core --cov-report=term-missing --cov-report=xml:build/coverage/coverage-python.xml - @echo "🛠️ Building and running C++ tests with coverage..." - @BUILD_DIR="build/cpp-tests-cov"; \ - mkdir -p "$$BUILD_DIR"; \ - GEN_ARGS=""; \ - if command -v ninja >/dev/null 2>&1; then GEN_ARGS="-G Ninja"; fi; \ - cmake -S . -B "$$BUILD_DIR" -DNETGRAPH_CORE_BUILD_TESTS=ON -DNETGRAPH_CORE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug $$GEN_ARGS; \ - cmake --build "$$BUILD_DIR" --config Debug -j; \ - ctest --test-dir "$$BUILD_DIR" --output-on-failure || echo "⚠️ Some C++ tests failed (continuing for coverage)" - @echo "📈 Generating C++ coverage XML (gcovr)..." - @$(PYTHON) -m gcovr --root . \ - --object-directory build \ - --object-directory build/cpp-tests-cov \ - --filter 'include/netgraph' --filter 'src' --exclude 'tests' --exclude 'bindings/.*' --exclude '.*pybind11.*' --exclude '_deps/pybind11-src/.*' \ - --gcov-ignore-errors=all \ - --gcov-ignore-parse-errors=negative_hits.warn_once_per_file \ - --xml-pretty -o build/coverage/coverage-cpp.xml - @echo "" - @echo "================ Python + C++ coverage (summary) ================" - @$(PYTHON) dev/coverage_summary.py build/coverage/coverage-python.xml build/coverage/coverage-cpp.xml --html=build/coverage/coverage-combined.html - @echo "" - @echo "✅ Coverage ready in build/coverage/: coverage-python.xml, coverage-cpp.xml, coverage-combined.html" + @$(PYTHON) -I -B dev/test_coverage_summary.py + @PYTHON=$(PYTHON) bash dev/run-checks.sh cov sanitize-test: @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh sanitize +tsan-test: + @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh tsan + +stubs-check: + @PYTHON=$(PYTHON) bash dev/stubs/run.sh check + +perf-tool-test: + @$(PYTHON) -B dev/perf/test_tool.py + # Clean + reinstall in dev mode (respects CMAKE_ARGS and MACOSX_DEPLOYMENT_TARGET) # Uses active PYTHON (venv or PATH) to avoid environment mismatches rebuild: clean diff --git a/README.md b/README.md index 6bd2eff..256eb0a 100644 --- a/README.md +++ b/README.md @@ -2,83 +2,189 @@ C++ graph engine for network flow analysis, traffic engineering simulation, and capacity planning. -## Overview +> **Unreleased, breaking.** This version replaces the 0.11 API (see the Unreleased section of +> [CHANGELOG.md](CHANGELOG.md)): the graph on integer units, the minimum-cost flow core, the route +> structures (`PredDAG`, `Path`), the integer ledger `FlowLedger` with the placement kernels, the +> policy engine `FlowPolicy`, the searches (`spf`, `ksp`, `resolve_to_paths`), the `DagCache` in +> building and frozen modes, and the evaluators `capacity`, `batch_capacity` and `sensitivity`. +> The previous API (0.11 and earlier) is in git history and on PyPI. -NetGraph-Core provides a specialized graph implementation for networking problems. Key design priorities: +## Overview -- **Determinism**: Guaranteed reproducible edge ordering by (cost, src, dst). -- **Flow Modeling**: Native support for multi-commodity flow state, residual tracking, and ECMP/WCMP placement. -- **Performance**: Immutable CSR (Compressed Sparse Row) adjacency, and zero-copy NumPy views over mutable flow state. +- **Exact integer amounts**: a graph carries an explicit resolution `q` (a power of two in + `[2^-32, 2^32]`); capacities and flows are `int64` units of `q`. Physical values enter only + through `to_units` (round half up) and leave through `as_float`. +- **Positive costs**: edge costs are integers `>= 1`, so the shortest-path DAG (every edge on + some shortest path) has no cycle and does not depend on search order or tie rules. +- **Stable edge ids**: edges are numbered by sorting on `(src, cost, dst)`, ties kept in input + order; that number is the public `EdgeId`, and `input_index()` maps it back to the input position. +- **One flow core**: an exact integer minimum-cost flow solver (the most traffic, sent as cheaply + as possible). It returns the flow value, the marginal cost curve (the extra minimum total cost + per additional delivered unit, as `(cost, units)` segments), the flow on every edge, a minimum + cut (the full edges that separate the sources from the destinations) and the node potentials + (per-node prices that prove the flow is cheapest). It works one cost level at a time: shortest + paths with potentials find the next cheapest cost, then Dinic phases fill every path of that + cost. ## Core Features -### 1. Graph Representations - -- **`StrictMultiDiGraph`**: Immutable directed multigraph using CSR adjacency. Supports parallel edges (multi-graph), essential for network topologies. -- **`FlowGraph`**: Topology overlay managing mutable flow state, per-flow edge allocations, and residual capacities. - -### 2. Network Algorithms - -- **Shortest Paths (SPF)**: - - Modified Dijkstra returns a **Predecessor DAG** to capture all equal-cost paths. - - Supports **ECMP** (Equal-Cost Multi-Path) routing. - - Features **node/edge masking** and **residual-aware tie-breaking**. - -- **K-Shortest Paths (KSP)**: - - Yen's algorithm returning DAG-wrapped paths. - - Configurable constraints on cost factors (e.g., paths within 1.5x of optimal). - -- **Explicit Paths**: - - `PredDAG.from_edges(graph, edges)` builds a path bundle from an operator-supplied - edge sequence, usable anywhere a `PredDAG` is accepted. - -- **Max-Flow**: - - **Algorithm**: Iterative augmentation using Successive Shortest Path on residual graphs, pushing flow across full ECMP/WCMP DAGs at each step. For `Proportional` placement with `require_capacity=True`, a final residual completion phase augments with reverse arcs so the result is a true maximum flow; the other modes are placement models and may report less. - - **Traffic Engineering (TE) Mode**: Routing adapts to residual capacity (progressive fill). - - **IP Routing Mode**: Cost-only routing (ECMP/WCMP) ignoring capacity constraints. - -- **Analysis**: - - **Sensitivity Analysis**: Identifies critical edges by removing each saturated edge and measuring how much total flow is *lost*. Supports `shortest_path` mode to analyze only edges used under ECMP routing (IP/IGP networks) vs. full max-flow (SDN/TE networks). - - **Min-Cut**: Computes minimum cuts on residual graphs. - -### 3. Flow Policy Engine - -Unified configuration object (`FlowPolicy`) that models diverse routing behaviors: - -- **Modeling**: Unified configuration for **IP Routing** (static costs) and **Traffic Engineering** (dynamic residuals). -- **Placement Strategies**: - - `EqualBalanced`: **ECMP** (equal splitting) - equal distribution across next-hops and parallel edges. - - `Proportional`: **WCMP** (weighted splitting) - distribution proportional to residual capacity. -- **Lifecycle Management**: Handles demand placement, re-optimization of existing flows, and constraints (path cost, stretch factor, flow counts). -- **Static (Pinned) Paths**: `FlowPolicy.set_static_paths()` pins a demand to explicit path bundles (MPLS-style). One flow per usable bundle; a bundle with no path surviving the failure masks is *down* and carries nothing, since a pinned path does not reroute. - -### 4. Python Integration - -- **Zero-Copy**: `FlowState` and `FlowGraph` `*_view()` methods expose C++ buffers as read-only NumPy arrays that track mutation in place. `StrictMultiDiGraph.*_view()` returns copies instead, so the graph's immutability cannot be violated. -- **Concurrency**: Releases the Python GIL during the long-running algorithms (SPF, KSP, max-flow, placement) to enable threading. `PredDAG.resolve_to_paths` holds the GIL. `batch_max_flow` and `sensitivity_analysis` also use internal worker threads, sized by `NGRAPH_CORE_BATCH_THREADS` / `NGRAPH_CORE_SENSITIVITY_THREADS`. +The exact behaviour of every call (units and rounding, edge ids, which route a search returns, +how the kernels split traffic, outcomes and flags, failures, the cache and the evaluators) is +specified in [docs/semantics.md](docs/semantics.md). + +### Graph + +- **`Graph(num_nodes, src, dst, capacity_units, cost, resolution)`**: immutable + directed multigraph that stores each node's out-edges and in-edges as arrays; parallel edges and + self-loops are allowed. Validation reports the first defect in a fixed order (the resolution, the unit + values, the shapes, zero or negative costs, the remaining bounds). Accessors (`capacity_units()`, `cost()`, `edge_src()`, `edge_dst()`, + `input_index()`, `row_offsets()`, `in_row_offsets()`, `in_adj_edge_index()`) return read-only + NumPy views that keep the graph alive. +- **`Eligibility`**: which nodes and edges of one graph may be used (immutable), built by + `graph.eligibility(excluded_nodes, excluded_edges)`; `Eligibility.intersect(a, b)` returns an + operand when the result equals it, otherwise a new value. + +### Route structures + +- **`PredDAG`**: the route structure a search returns, a predecessor DAG (directed, no cycle): + for each node, the edges that reach it on the route and the node each comes from. `parents()` + and `via_edges()` list these entries, in ascending EdgeId within a node; `parent_offsets()` + (built on each call) says where each node's entries start. + `PredDAG.from_csr(graph, offsets, parents, via)` validates the shape, the ranges and endpoints, + duplicates, acyclicity and the canonical order before accepting a DAG. +- **`Path(graph, hops)`**: a route given hop by hop, where a hop may hold several parallel edges + between the same two nodes; + `cost()` is the sum over hops of the minimum member cost and `to_dag()` gives the equivalent + `PredDAG`. +- **`PathSelection(tie_break, priority, multipath=True, multi_edge=True)`** with + `EdgeTieBreak` and `EdgePriority`: the selection axes the searches and policies read. + +### Ledger and kernels + +- **`FlowLedger(graph, residual=None)`**: the integer ledger of one graph: a baseline + in units (replaced by `reset()`) and the records (`FlowRecord`: flow id, owner, endpoints, + placement, admission, bundle, route cost, offered, delivered, per-edge allocation and drops) + of the flows placed on it. `allocated()` is the exact sum of the records and `residual()` + the baseline minus it; `bottleneck(sources)` lists the full edges leaving the set of nodes + the sources can still reach. `reset()` and `copy()` are rejected while a policy is bound + to the ledger. Policies place and remove flows; `reset()` replaces an unbound ledger's + baseline and clears its records. +- **Kernels**, the rules that spread one flow over its route (`FlowPlacement`: `MAX_FLOW`, + `WEIGHTED`, `EQUAL_BALANCED`; `Admission`: `LOSSLESS`, `LOSSY`): `MAX_FLOW` re-solves the flow + with the flow core on its DAG, over the free capacity plus its own allocation; `WEIGHTED` splits + at every node in proportion to weights fixed when the flow is created (highest averages: + whole-unit shares as close to proportional as possible); `EQUAL_BALANCED` splits equally (round + robin); `LOSSY` forwards hop by hop and records what it drops. + +### Policies + +- **`FlowPolicy(ledger, sources, destinations, config, elig=None, cache=None)`**: the placement + engine for one demand, bound to its ledger. `FlowPolicyConfig(routing=..., placement=..., + admission=..., selection=...)` sets four independent choices, all explicit. The routing is one of: + - `CostOnly()`: static shortest paths, as routers forward with an IGP such as OSPF; + - `CapacityAware(flow_count=1, max_flow_count=None, max_path_cost=None, min_flow_bandwidth=0, + reoptimize=False, diminishing=None)`: tunnels that each search the free capacity (CSPF), with + an `Absolute` or `Stretch` cost ceiling, a `DiminishingReturns(window, fraction)` guard, and at + most `MAX_FLOWS_PER_CALL` (10,000) new flows per call; + - `Global()`: an ideal controller: the minimum-cost flow from all the policy's sources to its + destinations, on the free capacity plus its own allocation; + - `Pinned(bundles)`: fixed routes given as `PredDAG`s. +- **`place(volume, outputs=None)`** adds `volume` to the policy's request as one all-or-nothing + step (CostOnly and CapacityAware place source by source; Global re-solves what it already + delivers plus this request) and returns a `PlacementResult`: `placed`, `remaining`, `termination`, `local_outcomes`, + `routes_changed`, `weights_changed`, `drops`, `cost_distribution` (Global only), and, when + requested through `OutputOptions`, `edge_flows`, `residual`, `bottleneck` and (Global only) + `min_cut`. + `remove()` reverts the policy's flows; `records()`, `record(flow_id)`, `placed_demand()` and + `usable_bundles()` read its state. +- **Failure guarantee**: every public call either completes or raises one exception and leaves + every ledger, policy, cache and scratch buffer as it was, whatever fails (validation, memory, a + worker thread, building the Python result). Two limits: pybind11 does not check every + allocation when it constructs a bound object from Python or converts a returned value into a + new Python object outside the library's checked result construction, so running out of memory + exactly there may crash the process (Core state stays unchanged); and a `DagCache` passed to the + call: a building cache keeps the complete entries and statistics of the lookups that succeeded + before the failure, and a frozen cache keeps its entries and statistics unchanged, except that a + call raising `DagCacheMiss` counts one miss + ([docs/semantics.md](docs/semantics.md#8-failures-threads-and-lifetimes)). + +### Search + +`spf(graph, roots, SpfOptions(direction, selection, targets=None, residual=None, +min_residual=1, elig=None))` searches from a root set, forward or reverse. +- **Without targets:** it returns `costs` (`UNREACHABLE` for nodes not reached) and the full + DAG of tight edges, the edges that lie on some shortest path (one per node pair with + `multi_edge=False`). +- **With targets:** it stops at the nearest targets and returns `target_cost` and the DAG of + the walks between the roots and the nearest targets (for a reverse search, from a target to + a root). +- **With `multipath=False`:** it returns the canonical single path: the walk of lowest cost; + among those, under `PREFER_HIGHER_PRIORITY` only, the one of highest bottleneck (the smallest + priority value along the walk); then the fewest hops; then the lexicographically smallest + sequence of EdgeIds ([docs/semantics.md](docs/semantics.md#the-canonical-single-path)). +- **With `multi_edge=False`** (any search): each node pair is first reduced to one parallel + member (lowest cost, then under `PREFER_HIGHER_PRIORITY` the highest priority, then the lowest + EdgeId). + +`ksp(graph, src, dst, KspOptions(k, selection, max_path_cost=None, residual=None, elig=None))` +returns the K shortest simple paths as `(cost, Path)` pairs, in the order of +[docs/semantics.md](docs/semantics.md#k-shortest-paths) and up to +an inclusive cost ceiling. `resolve_to_paths(graph, dag, src, dst, split_parallel_edges=False, +max_paths=None)` lists a DAG's walks as `Path` values. + +### DagCache + +`DagCache(graph)` keeps the static shortest-path DAGs (from costs and the Eligibility, not +traffic) that policies build, keyed by the root set, the direction and the Eligibility. Values +are immutable and shared, and `stats()` reports hits, misses, entries, retained bytes and live +values. +- **Building mode (new caches):** one writer; a miss searches and inserts. An evaluator given a + building cache runs on one worker (an explicit `workers` above 1 is rejected). +- **Frozen mode (`freeze()`):** read-only and shared by every worker of an evaluator (pass + `workers` explicitly; with a cache the default is 1); a miss raises `DagCacheMiss`. `thaw()` + returns to building mode once every reader has finished. + +### Evaluators + +`capacity(graph, sources, destinations, CapacityOptions(config, elig=None, cache=None, +outputs=None))` is `place(+inf)` of a policy on a fresh ledger; Global with MAX_FLOW gives the +exact maximum flow with its marginal cost curve, and its minimum cut with +`outputs=OutputOptions(bottleneck=True)`. `batch_capacity(graph, pairs, +opts, per_pair_elig=None, workers=None)` evaluates pairs in pair order (the lowest failing +index raises). `sensitivity(graph, sources, destinations, SensitivityOptions(...), +workers=None)` reports, for every candidate edge, the signed loss in placed units when the +edge is removed. With `workers=None` the worker count is 1 when a cache is supplied, otherwise +it comes from `NGRAPH_CORE_BATCH_THREADS` and `NGRAPH_CORE_SENSITIVITY_THREADS`. + +### Flow core + +One native integer minimum-cost flow core serves Global routing, the MAX_FLOW kernel and the +evaluators: the largest flow up to a cap from a source set to a destination set over the eligible +edges, sent at the lowest total cost, starting from zero flow. For sensitivity it repairs the +baseline's solution for each removed edge instead of solving again. Its private Python entry +`_netgraph_core._place_global_units(graph, view, sources, destinations, elig, cap)` returns the +payload `{placed, cost_distribution, edge_flows, min_cut, potentials, counters}` for the +flow checks of the test suites. ## Quick Start ```python -import numpy as np import netgraph_core as ngc -# Two parallel paths from node 0 to node 3. -graph = ngc.StrictMultiDiGraph.from_arrays( - num_nodes=4, - src=np.array([0, 1, 0, 2], dtype=np.int32), - dst=np.array([1, 3, 2, 3], dtype=np.int32), - capacity=np.array([10.0, 10.0, 5.0, 5.0], dtype=np.float64), - cost=np.array([1, 1, 1, 1], dtype=np.int64), - ext_edge_ids=np.arange(4, dtype=np.int64), # your own stable edge ids -) - -algs = ngc.Algorithms(ngc.Backend.cpu()) -handle = algs.build_graph(graph) +q = 2.0**-12 +# Two parallel paths from node 0 to node 3 (physical capacities 10 and 5). +caps = ngc.to_units([10.0, 10.0, 5.0, 5.0], q) +graph = ngc.Graph(4, [0, 1, 0, 2], [1, 3, 2, 3], caps, [1, 1, 2, 2], q) -total, summary = algs.max_flow(handle, 0, 3) -print(total) # 15.0 -print(summary.min_cut.edges.tolist()) # [0, 1] -- bottleneck edge ids +config = ngc.FlowPolicyConfig( + routing=ngc.Global(), + placement=ngc.FlowPlacement.MAX_FLOW, + admission=ngc.Admission.LOSSLESS, + selection=ngc.PathSelection(ngc.EdgeTieBreak.DETERMINISTIC, ngc.EdgePriority.CAPACITY), +) +result = ngc.capacity(graph, 0, 3, ngc.CapacityOptions(config)) +print(ngc.as_float(result.placed, q)) # 15.0 +print(result.cost_distribution) # [(2, 40960), (4, 20480)] ``` ## Installation @@ -87,12 +193,15 @@ print(summary.min_cut.edges.tolist()) # [0, 1] -- bottleneck edge ids pip install netgraph-core ``` -Or from source: +Or from source (the shipped configuration; `pip install -e .` is the editable development build, +which carries the private test hooks): ```bash -pip install -e . +pip install . ``` +The package is typed: it ships `py.typed` and a type stub generated from the compiled module. + ### Build Optimizations Default builds include LTO and loop unrolling. For local development: @@ -111,32 +220,41 @@ python/netgraph_core/ # Python package tests/cpp/ # C++ tests (googletest) tests/py/ # Python tests (pytest) dev/ # Developer scripts (not shipped): checks, coverage, -dev/perf/ # performance tooling and recorded A/B/A results, -dev/research/ # research prototypes and their evidence +dev/perf/ # and the A/B/A timing tool ``` ## Development ```bash -make dev # Setup: venv, dependencies, pre-commit hooks -make check # Run all tests and linting (auto-fix formatting) -make check-ci # Strict checks without auto-fix (for CI) -make test # Python tests -make cpp-test # C++ tests only -make cov # Combined coverage report (C++ + Python) +make dev # Setup: venv, dependencies, pre-commit hooks +make check # Rebuild the editable extension, then everything check-ci runs +make check-ci # Lint, types, C++ tests (hooks on and off), Python tests (CI entrypoint) +make test # Python tests +make cpp-test # C++ tests (private hooks on) +make cpp-test-off # C++ tests without the hooks (the shipped configuration) +make sanitize-test # C++ tests under ASan/UBSan +make tsan-test # C++ tests under ThreadSanitizer +make cov # Combined coverage report (C++ + Python; its own venv and builds) +make stubs-check # The generated type stub: drift, package contents, typed clients +make format # Apply ruff formatting (make hooks runs every pre-commit fixer) ``` -Performance work (harnesses, old/new/old protocol, recorded measurements) is -described in [dev/perf/README.md](dev/perf/README.md). +`make check` and `make check-ci` only verify; they never edit a file. + +Performance claims are timed with the A/B/A tool (old build, new build, then the old build again +on one machine, so drift shows), described in [dev/perf/README.md](dev/perf/README.md) +(`make perf-tool-test` tests the tool itself; CI runs it on one macOS job). + +The editable development build (`make install`, `make dev`) carries the private test hooks +(CMake option `NETGRAPH_CORE_PRIVATE_HOOKS`; a wheel build does not); the test suites skip the +tests that need them when they are absent. ## Environment Variables | Variable | Effect | | --- | --- | -| `NGRAPH_CORE_PROFILE=1` | Enable profiling of C++ hot paths (`profiling_dump()` / `profiling_reset()`). | -| `NGRAPH_CORE_BATCH_THREADS` | Worker threads for `batch_max_flow` (default: hardware concurrency). Set to `1` when calling from your own worker pool. | -| `NGRAPH_CORE_SENSITIVITY_THREADS` | Worker threads for `sensitivity_analysis` (default: hardware concurrency). | -| `NGRAPH_CORE_SPF_QUEUE` | Frontier queue for SPF: `heap` forces the reference binary heap, `bucket` prefers the bucket queue where the graph is eligible. Unset (default) selects automatically; both give bit-identical results. Read once per process; for benchmarking and diagnosis. | +| `NGRAPH_CORE_BATCH_THREADS` | Default worker count of `batch_capacity` (an integer in [1, 4096]; 1 when unset or invalid; not read when a cache is supplied, which defaults to 1; an explicit `workers` wins). | +| `NGRAPH_CORE_SENSITIVITY_THREADS` | Default worker count of `sensitivity` (as above). | ## Requirements diff --git a/bindings/python/axes.cpp b/bindings/python/axes.cpp new file mode 100644 index 0000000..ca26f56 --- /dev/null +++ b/bindings/python/axes.cpp @@ -0,0 +1,89 @@ +// The enumerations of the placement and selection axes, and PathSelection. +#include + +#include + +#include "netgraph/core/options.hpp" + +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_axes(py::module_& m) { + py::enum_(m, "FlowPlacement", + "How one flow spreads over its route (its kernel). MAX_FLOW: the minimum-cost flow core, re-solved\n" + "on the flow's route over the free capacity plus the flow's own allocation (it may move its own\n" + "earlier placement, never another flow's). WEIGHTED (WCMP, weighted-cost multipath): every node\n" + "splits in proportion to weights fixed when the flow is created (highest averages: whole-unit\n" + "shares as close to proportional as possible). EQUAL_BALANCED (ECMP, equal-cost multipath): every\n" + "node splits equally, by round robin.") + .value("MAX_FLOW", FlowPlacement::MaxFlow) + .value("WEIGHTED", FlowPlacement::Weighted) + .value("EQUAL_BALANCED", FlowPlacement::EqualBalanced); + + py::enum_(m, "Admission", + "LOSSLESS admits only what fits; LOSSY forwards hop by hop without admission control and\n" + "drops what does not fit on each edge (EQUAL_BALANCED and WEIGHTED only).") + .value("LOSSLESS", Admission::Lossless) + .value("LOSSY", Admission::Lossy); + + py::enum_(m, "EdgeTieBreak", + "How a search breaks cost ties. Between parallel members of one (u, v) pair\n" + "(multi_edge = False): DETERMINISTIC keeps the lowest EdgeId, PREFER_HIGHER_PRIORITY the\n" + "higher priority value, then the lowest EdgeId. Between equal-cost walks (multipath =\n" + "False): PREFER_HIGHER_PRIORITY first takes the highest bottleneck (the smallest priority\n" + "value along the walk), DETERMINISTIC skips that step; then the fewest hops, then the\n" + "lexicographically smallest EdgeId sequence.") + .value("DETERMINISTIC", EdgeTieBreak::Deterministic) + .value("PREFER_HIGHER_PRIORITY", EdgeTieBreak::PreferHigherPriority); + + py::enum_(m, "EdgePriority", + "The per-edge value PREFER_HIGHER_PRIORITY reads and, with WEIGHTED, the value that decides which\n" + "edges a flow uses and their weights: RESIDUAL (residual units), CAPACITY (static capacity units),\n" + "HEADROOM (residual / capacity, compared exactly as a fraction; rejected with WEIGHTED).") + .value("RESIDUAL", EdgePriority::Residual) + .value("CAPACITY", EdgePriority::Capacity) + .value("HEADROOM", EdgePriority::Headroom); + + py::enum_(m, "Direction", + "The orientation of a search: FORWARD from the roots along the edges, or REVERSE toward the roots\n" + "against the edges (the DAG it returns keeps the edges' own direction, so its walks lead to the\n" + "roots).") + .value("FORWARD", Direction::Forward) + .value("REVERSE", Direction::Reverse); + + py::class_(m, "PathSelection", + "The selection of a search: tie_break and priority are explicit; multipath = True keeps\n" + "every equal-cost walk (the DAG), False the one canonical single path; multi_edge = True\n" + "keeps every eligible parallel member, False one per (u, v) pair. A tie_break or priority\n" + "outside its enumeration raises ValueError.") + // The constructor checks both enumerations (ValueError naming the field). + .def(py::init(), py::arg("tie_break"), py::arg("priority"), + py::arg("multipath") = true, py::arg("multi_edge") = true) + .def_readonly("tie_break", &PathSelection::tie_break, "the EdgeTieBreak") + .def_readonly("priority", &PathSelection::priority, "the EdgePriority") + .def_readonly("multipath", &PathSelection::multipath, + "True: every equal-cost walk (a DAG); False: the canonical single path") + .def_readonly("multi_edge", &PathSelection::multi_edge, + "True: every eligible parallel member; False: one member per (u, v) pair") + .def("__eq__", [](const PathSelection& a, const py::object& b) { + return py::isinstance(b) && a == b.cast(); + }) + .def("__hash__", [](const PathSelection& a) { + return py::hash(py::make_tuple(static_cast(a.tie_break), static_cast(a.priority), a.multipath, + a.multi_edge)); + }) + .def("__repr__", [](const PathSelection& a) { + return std::string("PathSelection(tie_break=") + + (a.tie_break == EdgeTieBreak::Deterministic ? "DETERMINISTIC" : "PREFER_HIGHER_PRIORITY") + + ", priority=" + + (a.priority == EdgePriority::Residual ? "RESIDUAL" + : (a.priority == EdgePriority::Capacity ? "CAPACITY" : "HEADROOM")) + + ", multipath=" + (a.multipath ? "True" : "False") + ", multi_edge=" + (a.multi_edge ? "True" : "False") + + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/dag_cache.cpp b/bindings/python/dag_cache.cpp new file mode 100644 index 0000000..1e5252b --- /dev/null +++ b/bindings/python/dag_cache.cpp @@ -0,0 +1,60 @@ +// DagCacheStats, DagCacheMiss and DagCache: the cache of static shortest-path DAGs. +#include +#include +#include + +#include + +#include "netgraph/core/dag_cache.hpp" +#include "netgraph/core/graph.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_dag_cache(py::module_& m) { + py::class_(m, "DagCacheStats", + "DagCache statistics (atomic per field, not a consistent snapshot): hits, misses,\n" + "entries, bytes (the entries' retained payload), live_values (values alive anywhere).") + .def_readonly("hits", &DagCache::Stats::hits, "lookups answered from an entry (they persist across clear())") + .def_readonly("misses", &DagCache::Stats::misses, + "lookups without an entry: a building cache searched, a frozen one raised DagCacheMiss") + .def_readonly("entries", &DagCache::Stats::entries, "the entries held") + .def_readonly("bytes", &DagCache::Stats::bytes, "the retained payload of the entries (their DAGs' stored arrays)") + .def_readonly("live_values", &DagCache::Stats::live_values, "the cached values alive anywhere, entries included"); + + py::register_exception(m, "DagCacheMiss", PyExc_Exception); + m.attr("DagCacheMiss").attr("__doc__") = + "A lookup in a frozen DagCache whose key is absent: a dedicated exception (not a\n" + "RuntimeError or ValueError); the evaluators propagate it unchanged."; + + py::class_>( + m, "DagCache", + "A cache of static shortest-path DAGs (from costs and the Eligibility only) for one graph, keyed by\n" + "(the root set, the direction, the Eligibility uid). It stores the full DAG and applies each\n" + "caller's selection afterwards, so the selection is never cached. CostOnly policies and the\n" + "evaluators read it.\n" + "Building mode (a new cache): one writer, no concurrent readers; a miss searches and inserts;\n" + "clear() empties it (the hit and miss counters persist), and values already handed out stay valid.\n" + "Frozen mode (freeze()): read-only and shareable by every worker; a miss counts one miss and raises\n" + "DagCacheMiss; clear() raises ValueError; thaw() returns to building once the caller has joined\n" + "every reader.\n" + "When a public call fails: a building cache passed to it keeps the complete entries and statistics\n" + "of the lookups that succeeded before the failure (never a partial entry; a kept entry never\n" + "changes a result); a frozen cache's statistics stay unchanged, except one miss when the call\n" + "raised DagCacheMiss. A frozen cache counts its hits only when the call succeeds, Python result\n" + "included.") + .def(py::init([](const std::shared_ptr& graph) { + if (!graph) throw py::value_error("graph must be a Graph"); + return std::make_shared(*graph); + }), + py::arg("graph"), "An empty cache in building mode, bound to graph (by its uid).") + .def("graph_uid", &DagCache::graph_uid, "uid of the graph this cache is bound to") + .def("freeze", &DagCache::freeze, "enter frozen mode (no effect when frozen)") + .def("thaw", &DagCache::thaw, "return to building mode (no effect when building); join every reader first") + .def("clear", &DagCache::clear, "empty a building cache; ValueError when frozen") + .def("stats", &DagCache::stats, "the current DagCacheStats (each field read atomically)"); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/eligibility.cpp b/bindings/python/eligibility.cpp new file mode 100644 index 0000000..bcb96b7 --- /dev/null +++ b/bindings/python/eligibility.cpp @@ -0,0 +1,54 @@ +// Eligibility: the static node and edge flags of one graph. +#include +#include +#include + +#include +#include + +#include "netgraph/core/eligibility.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_eligibility(py::module_& m) { + py::class_>( + m, "Eligibility", + "Which nodes and edges of one graph may be used: one flag per node and per edge, fixed when built\n" + "(it does not depend on traffic). Built by Graph.eligibility(excluded_nodes, excluded_edges) and\n" + "Eligibility.intersect.") + .def("graph_uid", &Eligibility::graph_uid, "uid of the graph this value belongs to") + .def("uid", &Eligibility::uid, "0 for the all-allowed value, a fresh uid otherwise") + .def( + "node_allowed", + [](const py::object& self) { + const auto& el = self.cast(); + return readonly_view(el.node_allowed(), self, py::dtype("bool")); + }, + "bool[N] read-only view") + .def( + "edge_allowed", + [](const py::object& self) { + const auto& el = self.cast(); + return readonly_view(el.edge_allowed(), self, py::dtype("bool")); + }, + "bool[E] read-only view (not excluded and both endpoints allowed)") + .def_static( + "intersect", + [](const std::shared_ptr& a, const std::shared_ptr& b) -> std::shared_ptr { + if (!a || !b) throw py::value_error("Eligibility.intersect takes two Eligibility values"); + auto r = Eligibility::intersect(*a, *b); + if (r.uid() == a->uid()) return a; + if (r.uid() == b->uid()) return b; + return std::make_shared(std::move(r)); + }, + py::arg("a"), py::arg("b"), + "The intersection of two values of one graph (ValueError for values of different graphs):\n" + "an operand itself when the result equals it (a first, then b), otherwise a new value with a\n" + "fresh uid; equal intersections built separately get distinct uids."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/evaluators.cpp b/bindings/python/evaluators.cpp new file mode 100644 index 0000000..902e250 --- /dev/null +++ b/bindings/python/evaluators.cpp @@ -0,0 +1,241 @@ +// capacity, batch_capacity and sensitivity with their options and results. Each evaluator +// places on a new, empty ledger; a frozen DagCache counts its hits only when the whole Python +// call succeeds (detail::frozen_call). +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/dag_cache.hpp" +#include "netgraph/core/detail/evaluator_checks.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/evaluators.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The Python form of CapacityOptions and SensitivityOptions: the Eligibility and the cache are +// held by reference count for the call (the C++ structs borrow raw pointers). +struct PyCapacityOptions { + FlowPolicyConfig config; + std::shared_ptr elig; + std::shared_ptr cache; + OutputOptions outputs; +}; +struct PySensitivityOptions { + FlowPolicyConfig config; + std::shared_ptr elig; + std::shared_ptr cache; + OutputOptions outputs; +}; + +// workers: an explicit integer is passed through (validated by the evaluator); None takes the +// environment default (detail::default_workers), and 1 whenever a cache is supplied (a building +// cache has one writer; a frozen one is shared only when the caller asks for workers). +std::int32_t workers_of(const py::object& workers, bool sensitivity, bool cache_given) { + if (!workers.is_none()) { + return static_cast(integer_value(workers, std::numeric_limits::min(), + std::numeric_limits::max(), "workers")); + } + if (cache_given) return 1; + return detail::default_workers(sensitivity); +} + +} // namespace + +void register_evaluators(py::module_& m) { + py::class_(m, "CapacityOptions", + "The configuration, the static Eligibility (None: all allowed), the DagCache (None:\n" + "private per worker) and the optional outputs of capacity() and batch_capacity().") + .def(py::init([](const FlowPolicyConfig& config, const std::shared_ptr& elig, + const std::shared_ptr& cache, const py::object& outputs) { + return PyCapacityOptions {config, elig, cache, outputs_of(outputs)}; + }), + py::arg("config"), py::kw_only(), py::arg("elig") = nullptr, py::arg("cache") = nullptr, + py::arg("outputs") = py::none()) + .def_readonly("config", &PyCapacityOptions::config, "the FlowPolicyConfig of every evaluated policy") + .def_readonly("elig", &PyCapacityOptions::elig, "the static Eligibility; None: everything allowed") + .def_readonly("cache", &PyCapacityOptions::cache, "the supplied DagCache; None: a private cache per worker") + .def_readonly("outputs", &PyCapacityOptions::outputs, "the OutputOptions of every result"); + + py::class_(m, "SensitivityOptions", + "The configuration, Eligibility, DagCache and baseline outputs of sensitivity().") + .def(py::init([](const FlowPolicyConfig& config, const std::shared_ptr& elig, + const std::shared_ptr& cache, const py::object& outputs) { + return PySensitivityOptions {config, elig, cache, outputs_of(outputs)}; + }), + py::arg("config"), py::kw_only(), py::arg("elig") = nullptr, py::arg("cache") = nullptr, + py::arg("outputs") = py::none()) + .def_readonly("config", &PySensitivityOptions::config, "the FlowPolicyConfig of the baseline and every deletion") + .def_readonly("elig", &PySensitivityOptions::elig, "the static Eligibility; None: everything allowed") + .def_readonly("cache", &PySensitivityOptions::cache, "the supplied DagCache; None: a private cache per worker") + .def_readonly("outputs", &PySensitivityOptions::outputs, "the OutputOptions of the baseline result"); + + py::class_(m, "SensitivityEntry", + "One candidate edge: edge and loss (int units: the baseline minus the capacity without the\n" + "edge).") + .def_readonly("edge", &SensitivityEntry::edge) + .def_readonly("loss", &SensitivityEntry::loss); + + py::class_(m, "SensitivityResult", + "baseline (the PlacementResult of capacity()) and entries (ascending EdgeId; an\n" + "absent edge has loss 0).") + .def_readonly("baseline", &SensitivityResult::baseline) + .def_readonly("entries", &SensitivityResult::entries); + + m.attr("MAX_FLOWS_PER_CALL") = py::int_(kMaxFlowsPerCall); + + m.def( + "capacity", + [](const std::shared_ptr& graph, const py::object& sources, const py::object& destinations, + const py::object& opts) { + if (!graph) throw py::value_error("graph must be a Graph"); + const auto o = options_of(opts, "CapacityOptions"); + const auto s = node_ids(sources, graph->num_nodes(), "sources"); + const auto d = node_ids(destinations, graph->num_nodes(), "destinations"); + const CapacityOptions c {o.config, o.elig.get(), o.cache.get(), o.outputs}; + // One public call on a frozen cache's statistics, its result object included: its hits count + // only when the Python result exists. + return detail::frozen_call(c.cache, [&](detail::FrozenTally& tally) { + PlacementResult r; + { + py::gil_scoped_release release; + r = detail::capacity_in_call(*graph, s, d, c, tally); + } + return object_of(std::move(r)); + }); + }, + py::arg("graph"), py::arg("sources"), py::arg("destinations"), py::arg("opts"), + "place(+inf) of a FlowPolicy with opts.config on a fresh ledger of the graph -> PlacementResult.\n" + "The exact maximum flow is the Global + MAX_FLOW configuration. A supplied frozen cache raises\n" + "DagCacheMiss for an absent key, unchanged.\n" + "A failed call raises; a supplied DagCache then behaves as the DagCache docstring states."); + + m.def( + "batch_capacity", + [](const std::shared_ptr& graph, const py::object& pairs, const py::object& opts, + const py::object& per_pair_elig, const py::object& workers) { + if (!graph) throw py::value_error("graph must be a Graph"); + // The arguments first, then each pair completely and in pair order (the same two parts + // as the C++ validation), so the lowest failing pair index is reported. + const auto o = options_of(opts, "CapacityOptions"); + const auto w = workers_of(workers, false, static_cast(o.cache)); + const CapacityOptions c {o.config, o.elig.get(), o.cache.get(), o.outputs}; + detail::check_batch_arguments(*graph, c, w); + if (not_a_sequence(pairs)) throw py::value_error("pairs must be a sequence of (sources, destinations)"); + const auto items = py::reinterpret_borrow(pairs); + const auto num_pairs = py::len(items); + std::optional eligs; + if (!per_pair_elig.is_none()) { + // An explicit sequence has one entry per pair (None entries allowed), also when empty. + if (not_a_sequence(per_pair_elig)) throw py::value_error("per_pair_elig must be a sequence of Eligibility or None"); + eligs = py::reinterpret_borrow(per_pair_elig); + if (py::len(*eligs) != num_pairs) { + throw py::value_error("per_pair_elig has " + std::to_string(py::len(*eligs)) + " entries for " + + std::to_string(num_pairs) + " pairs"); + } + } + std::vector, std::vector>> ps; + std::vector> keep; + std::vector per; + const auto n = graph->num_nodes(); + for (std::size_t i = 0; i < num_pairs; ++i) { + const py::object item = items[i]; + if (not_a_sequence(item) || py::len(item) != 2) { + throw py::value_error("each pair must be (sources, destinations)"); + } + auto t = py::reinterpret_borrow(item); + auto src = node_ids(t[0], n, "sources"); + auto dst = node_ids(t[1], n, "destinations"); + const Eligibility* e = nullptr; + if (eligs) { + const py::object entry = (*eligs)[i]; + if (!entry.is_none()) { + if (!py::isinstance(entry)) throw py::type_error("per_pair_elig entries must be Eligibility or None"); + keep.push_back(entry.cast>()); + e = keep.back().get(); + } + per.push_back(e); + } + detail::check_batch_pair(*graph, src, dst, e, i); + ps.emplace_back(std::move(src), std::move(dst)); + } + return detail::frozen_call(c.cache, [&](detail::FrozenTally& tally) { + std::vector rs; + { + py::gil_scoped_release release; + rs = detail::batch_capacity_in_call(*graph, ps, c, per, w, tally); + } + py::list out(rs.size()); + for (std::size_t i = 0; i < rs.size(); ++i) { + PyList_SET_ITEM(out.ptr(), static_cast(i), object_of(std::move(rs[i])).release().ptr()); + } + return py::object(std::move(out)); + }); + }, + py::arg("graph"), py::arg("pairs"), py::arg("opts"), py::arg("per_pair_elig") = py::none(), + py::arg("workers") = py::none(), + "capacity() for each (sources, destinations) pair, results in pair order. Pair i uses opts.elig\n" + "intersected with per_pair_elig[i] (None entries allowed; a given sequence has one entry per pair,\n" + "also when empty). workers >= 1; None: NGRAPH_CORE_BATCH_THREADS or 1, and 1 when a cache is\n" + "supplied. Everything is validated before a worker starts, the arguments first, then each pair\n" + "completely in pair order, so the error of the lowest failing pair is raised. Pinned is rejected;\n" + "an empty batch returns [].\n" + "Caches: a supplied building cache needs workers == 1; a frozen cache is shared by every worker\n" + "(pass workers explicitly), and its DagCacheMiss reaches the caller unchanged (the lowest failing\n" + "pair's). An intersection that differs from both operands gets a new Eligibility uid, which a\n" + "frozen cache cannot hold: pass the combined per-pair values with elig None, or a common value\n" + "that contains them.\n" + "A failed call raises; a supplied DagCache then behaves as the DagCache docstring states."); + + m.def( + "sensitivity", + [](const std::shared_ptr& graph, const py::object& sources, const py::object& destinations, + const py::object& opts, const py::object& workers) { + if (!graph) throw py::value_error("graph must be a Graph"); + const auto o = options_of(opts, "SensitivityOptions"); + const auto s = node_ids(sources, graph->num_nodes(), "sources"); + const auto d = node_ids(destinations, graph->num_nodes(), "destinations"); + const auto w = workers_of(workers, true, static_cast(o.cache)); + const SensitivityOptions c {o.config, o.elig.get(), o.cache.get(), o.outputs}; + return detail::frozen_call(c.cache, [&](detail::FrozenTally& tally) { + SensitivityResult r; + { + py::gil_scoped_release release; + r = detail::sensitivity_in_call(*graph, s, d, c, w, tally); + } + return object_of(std::move(r)); + }); + }, + py::arg("graph"), py::arg("sources"), py::arg("destinations"), py::arg("opts"), py::arg("workers") = py::none(), + "The baseline capacity() and one SensitivityEntry per candidate edge. Candidates: with Global, the\n" + "edges that carry flow in the baseline (removing any other edge cannot lower a maximum flow); with\n" + "the other routings, every allowed edge of at least 1 unit. loss = baseline.placed minus the placed\n" + "of capacity() without the edge, in signed units, never thresholded. Two shortcuts give exactly\n" + "that value: Global repairs a copy of the baseline's solve instead of solving again, and CostOnly\n" + "gives loss 0 without placing to an edge outside the static shortest-path support reachable from\n" + "the roster sources (no route can use it). workers as batch_capacity\n" + "(NGRAPH_CORE_SENSITIVITY_THREADS). Only CostOnly reads the cache, and each placed deletion uses a\n" + "new Eligibility uid, so a frozen cache raises DagCacheMiss there.\n" + "A failed deletion fails the call (the lowest failing candidate's exception, no partial result); a\n" + "supplied DagCache then behaves as the DagCache docstring states."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_ledger.cpp b/bindings/python/flow_ledger.cpp new file mode 100644 index 0000000..cbfb0ec --- /dev/null +++ b/bindings/python/flow_ledger.cpp @@ -0,0 +1,103 @@ +// FlowLedger: the integer ledger of one graph. +#include +#include +#include + +#include + +#include "netgraph/core/flow_ledger.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "values.hpp" +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include "netgraph/core/detail/ledger_access.hpp" +#endif +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_flow_ledger(py::module_& m) { + [[maybe_unused]] auto ledger_class = py::class_>( + m, "FlowLedger", + "The integer ledger of one graph: a baseline residual in units (replaced only by reset()) and the\n" + "records of the flows placed on it by policies (the ledger has no public placement or removal).\n" + "allocated() is the sum of every record's edges and residual() = baseline - allocated(), both\n" + "int64[E] snapshots. reset() and copy() are rejected while a policy is bound (a FlowPolicy built\n" + "on this ledger still exists).") + .def(py::init([](const std::shared_ptr& graph, const py::object& residual) { + if (!graph) throw py::value_error("graph must be a Graph"); + if (residual.is_none()) return std::make_shared(graph); + const auto r = edge_units(residual, *graph, "residual"); + return std::make_shared(graph, r); + }), + py::arg("graph"), py::arg("residual") = py::none(), + "residual: the baseline as integer units, one entry per edge in [0, capacity_units[e]] (a\n" + "physical baseline goes through to_units first); None: the capacities. ValueError otherwise.") + .def("uid", &FlowLedger::uid, "process-unique ledger uid (the first half of every FlowId)") + .def( + "graph", [](const FlowLedger& fg) { return std::const_pointer_cast(fg.graph()); }, + "the ledger's graph") + .def( + "capacity_units", + [](const py::object& self) { return readonly_view(self.cast().capacity_units(), self); }, + "int64[E] the graph's capacities in units (read-only view)") + .def( + "residual", [](const FlowLedger& fg) { return int64_array(fg.residual()); }, + "int64[E] snapshot: baseline - allocated") + .def( + "allocated", [](const FlowLedger& fg) { return int64_array(fg.allocated()); }, + "int64[E] snapshot: the sum of every record's edges") + .def("records", &FlowLedger::records, "live records as snapshots in FlowId order: those of bound policies and orphans (records left by a\n" + "policy destroyed without remove())") + .def( + "record", [](const FlowLedger& fg, const py::object& flow_id) { return fg.record(flow_id_of(flow_id)); }, + py::arg("flow_id"), + "one record by FlowId (ValueError for a FlowId of another ledger, a retired or an unknown id)") + .def( + "bottleneck", + [](const FlowLedger& fg, const py::object& sources, const std::shared_ptr& elig) { + const auto s = node_ids(sources, fg.graph()->num_nodes(), "sources"); + return fg.bottleneck(s, elig.get()); + }, + py::arg("sources"), py::arg("elig") = nullptr, + "Saturation report: the eligible edges with capacity >= 1 unit that leave the set of nodes the\n" + "allowed sources can reach (following eligible edges forward with residual >= 1 unit), in\n" + "ascending EdgeId. It ignores the destinations, so it is not a source-destination cut.") + .def( + "reset", + [](FlowLedger& fg, const py::object& residual) { + if (residual.is_none()) { + fg.reset(); + return; + } + const auto r = edge_units(residual, *fg.graph(), "residual"); + fg.reset(r); + }, + py::arg("residual") = py::none(), + "A new baseline (None: the capacities); orphaned records and their allocations are cleared\n" + "and their ids retired (ids are never reused). ValueError while a policy is bound.") + .def( + "copy", [](const FlowLedger& fg) { return std::make_shared(fg); }, + "a new ledger (new uid) with the same baseline and records, ids kept under the new uid;\n" + "ValueError while a policy is bound") + .def("__copy__", [](const FlowLedger& fg) { return std::make_shared(fg); }) + .def("__deepcopy__", [](const FlowLedger& fg, const py::object&) { return std::make_shared(fg); }, + py::arg("memo")); +#if NETGRAPH_CORE_PRIVATE_HOOKS + ledger_class + .def( + "_state", + [](const FlowLedger& fg) { + py::dict out; + out["next_flow_id"] = detail::LedgerAccess::next_flow_id(fg); + out["bound"] = detail::LedgerAccess::bound_owners(fg); + out["orphans"] = detail::LedgerAccess::orphan_owners(fg); + return out; + }, + "private: {next_flow_id, bound, orphans} (owner ids)"); +#endif +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_policy.cpp b/bindings/python/flow_policy.cpp new file mode 100644 index 0000000..33394b9 --- /dev/null +++ b/bindings/python/flow_policy.cpp @@ -0,0 +1,190 @@ +// FlowPolicy: the one placement engine, bound to its ledger. +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/dag_cache.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/flow_ledger.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "values.hpp" +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include + +#include "netgraph/core/detail/pred_dag_builder.hpp" +#include "netgraph/core/detail/ledger_access.hpp" +#include "netgraph/core/detail/policy_engine.hpp" +#include "netgraph/core/detail/private_hooks.hpp" +#include "trace.hpp" +#endif +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// A request: an integer number of units in [0, 2^62) or +inf (float('inf')); nullopt is +inf. +std::optional request_units(const py::object& volume) { + if (py::isinstance(volume) || py::isinstance(volume, py::module_::import("numpy").attr("floating"))) { + const double x = PyFloat_AsDouble(volume.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + if (std::isinf(x) && x > 0) return std::nullopt; + throw py::value_error("requests are integer units or +inf (a physical request goes through to_units first; got " + + std::string(py::str(volume)) + ")"); + } + const auto v = integer_value(volume, 0, kDomainLimit - 1, "request (units)"); + return static_cast(v); +} + +#if NETGRAPH_CORE_PRIVATE_HOOKS + +// The epoch state of a policy (roster, requested, +inf, per roster source the flows). +py::dict state_obj(const FlowPolicy& policy) { + const auto& pe = detail::PolicyAccess::engine(policy); + py::dict out; + const auto& r = pe.config.routing; + out["mode"] = std::holds_alternative(r) ? "CostOnly" + : std::holds_alternative(r) ? "CapacityAware" + : std::holds_alternative(r) ? "Global" + : "Pinned"; + out["roster"] = pe.roster; + out["requested"] = counter_int(pe.requested); + out["infinite"] = pe.infinite; + out["placed_demand"] = policy.placed_demand(); + py::dict cohorts; + for (std::size_t i = 0; i < pe.roster.size(); ++i) { + py::list flows; + for (const auto& s : pe.cohorts[i]) { + const FlowRecord& rec = detail::LedgerAccess::live(pe.fg, s.id); + py::dict f; + f["flow_id"] = py::make_tuple(rec.flow_id.first, rec.flow_id.second); + f["src"] = s.src; + if (s.plan) { + f["route"] = s.plan->route; + } else { + const auto edges = detail::PredDAGBuilder::ascending(*s.bundle); + f["route"] = std::vector(edges.begin(), edges.end()); + } + f["members"] = s.plan ? py::cast(s.plan->members) : py::object(py::none()); + f["weights"] = s.plan && s.plan->placement == FlowPlacement::Weighted ? py::cast(s.plan->weights) + : py::object(py::none()); + f["T"] = rec.delivered; + f["delivered"] = rec.delivered; + f["alloc"] = pairs_list(rec.edges); + f["bundle_index"] = s.bundle_index < 0 ? py::object(py::none()) : py::object(py::int_(s.bundle_index)); + flows.append(f); + } + cohorts[py::int_(pe.roster[i])] = flows; + } + out["cohorts"] = cohorts; + return out; +} + +#endif + +} // namespace + +void register_flow_policy(py::module_& m) { + [[maybe_unused]] auto flow_policy = py::class_( + m, "FlowPolicy", + "The placement engine for one demand, bound to its ledger at construction. place(volume, outputs)\n" + "is all-or-nothing: a failed call raises and leaves the ledger and the policy as they were (with\n" + "the two limits in the module docstring). remove() takes every flow of the policy back out and\n" + "starts a new epoch (the request total starts again from 0). A policy dropped without remove()\n" + "leaves its records in the ledger as orphans: readable, never changed again, cleared by\n" + "FlowLedger.reset(). sources and destinations are one NodeId or a sequence; the two sets must not\n" + "overlap. ValueError for invalid endpoints, configurations and requests (non-integer node ids and\n" + "requests included) and for objects of another graph, ledger or policy; TypeError for a ledger,\n" + "config, elig, cache or outputs of the wrong class (a ledger of None: ValueError) and for an\n" + "unknown keyword.") + .def(py::init([](const std::shared_ptr& ledger, const py::object& sources, + const py::object& destinations, const FlowPolicyConfig& config, + const std::shared_ptr& elig, const std::shared_ptr& cache) { + if (!ledger) throw py::value_error("ledger must be a FlowLedger"); + const auto n = ledger->graph()->num_nodes(); + const auto s = node_ids(sources, n, "sources"); + const auto d = node_ids(destinations, n, "destinations"); + return std::make_unique(*ledger, s, d, config, elig.get(), cache.get()); + }), + py::keep_alive<1, 2>(), py::keep_alive<1, 6>(), py::keep_alive<1, 7>(), py::arg("ledger"), + py::arg("sources"), py::arg("destinations"), py::arg("config"), py::arg("elig") = nullptr, + py::arg("cache") = nullptr, + "Bind a policy to ledger. elig (None: everything allowed) and cache (None: none; only CostOnly\n" + "routing reads it) belong to the ledger's graph; the policy keeps the ledger, elig and cache\n" + "alive. The roster (the sources that statically reach an allowed destination) and, for Pinned,\n" + "the usable bundles are fixed here. Everything is validated before the policy binds.") + .def( + "place", + [](FlowPolicy& p, const py::object& volume, const py::object& outputs) { + const auto v = request_units(volume); + const auto o = outputs_of(outputs); + // All or nothing across the boundary: the result's Python object exists before the + // call, and after the commit nothing allocates (the value moves into it). The GIL + // stays held: the call mutates the ledger (one writer per ledger). + auto [obj, slot] = new_object(); + *slot = p.place(v, o); + return std::move(obj); + }, + py::arg("volume"), py::arg("outputs") = py::none(), + "Place volume (int units in [0, 2^62), or float('inf')) all or nothing -> PlacementResult.\n" + "Requests add up within an epoch (from construction or the last remove()); after float('inf') any\n" + "finite request is rejected, 0 included. A request of 0 places nothing. outputs (OutputOptions;\n" + "None: none) requests edge_flows, residual and bottleneck. Everything is validated before any\n" + "change; a failed call raises with the ledger and the policy unchanged. A solve of the flow core\n" + "(Global, or a MAX_FLOW flow) on a network beyond its int32 numbering raises ValueError: more than\n" + "INT32_MAX - 2 nodes, or more than 2^30 - 1 edges, sources and destinations together.") + .def("remove", &FlowPolicy::remove, "revert every flow of the policy, retire the ids, open a new epoch") + .def("placed_demand", &FlowPolicy::placed_demand, "the sum of delivered over the policy's live records") + .def("records", &FlowPolicy::records, "the policy's live records as snapshots in FlowId order") + .def( + "record", [](const FlowPolicy& p, const py::object& flow_id) { return p.record(flow_id_of(flow_id)); }, + py::arg("flow_id"), + "one of the policy's live records by FlowId (ValueError for a FlowId of another ledger or policy, a\n" + "retired or an unknown id)") + .def("usable_bundles", &FlowPolicy::usable_bundles, + "Pinned: the bundles with a src -> dst walk over eligible edges of at least 1 unit of capacity (0\n" + "for the other modes)") + .def_property_readonly("config", &FlowPolicy::config, "the FlowPolicyConfig the policy was built with"); +#if NETGRAPH_CORE_PRIVATE_HOOKS + flow_policy + .def( + "_place_traced", + [](FlowPolicy& p, const py::object& volume, const py::object& outputs) { + const auto v = request_units(volume); + const auto o = outputs_of(outputs); + // As place(): every Python object exists before anything changes; the engine runs the + // `staged` callback, which converts the trace, after the work is done and before the + // commit, so a failed conversion changes nothing. + auto [obj, slot] = new_object(); + py::tuple pair(2); + py::object trace_py; + detail::Trace trace; + const std::function staged = [&] { trace_py = trace_obj(trace); }; + *slot = detail::place_traced(p, v, o, trace, &staged); + PyTuple_SET_ITEM(pair.ptr(), 0, obj.release().ptr()); + PyTuple_SET_ITEM(pair.ptr(), 1, trace_py.release().ptr()); + return pair; + }, + py::arg("volume"), py::arg("outputs") = py::none(), + "private: place() with the transaction's trace (the call's intermediate values, keyed as\n" + "the tests' model of the policy keys them)") + .def("_state", &state_obj, "private: the epoch state (roster, requested, +inf, cohorts)") + .def( + "_owner", [](const FlowPolicy& p) { return detail::PolicyAccess::engine(p).owner; }, + "private: the ledger-local owner id (FlowRecord.owner)"); +#endif +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_record.cpp b/bindings/python/flow_record.cpp new file mode 100644 index 0000000..ff85348 --- /dev/null +++ b/bindings/python/flow_record.cpp @@ -0,0 +1,73 @@ +// FlowRecord: one flow of a ledger, as a snapshot. +#include +#include +#include + +#include +#include + +#include "netgraph/core/flow_record.hpp" +#include "netgraph/core/pred_dag.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +py::object cost_or_none(Cost c) { return c == kUnreachable ? py::object(py::none()) : py::object(py::int_(c)); } + +} // namespace + +void register_flow_record(py::module_& m) { + py::class_( + m, "FlowRecord", + "One flow of a ledger, as a snapshot: flow_id (ledger uid, id), owner (the policy's id),\n" + "sources, destinations, placement, admission, bundle (PredDAG: the flow's route),\n" + "route_cost (its minimum walk cost; None when empty), offered (Lossy only, else None),\n" + "delivered, edges [(EdgeId, units)] and drops [(EdgeId, units)] (cumulative, ascending\n" + "EdgeId). offered and the drops units are exact Python ints (they may exceed 64 bits).") + .def_property_readonly( + "flow_id", [](const FlowRecord& r) { return py::make_tuple(r.flow_id.first, r.flow_id.second); }, + "(ledger uid, id): ids are assigned in creation order and never reused in the ledger") + .def_readonly("owner", &FlowRecord::owner, "the ledger-local id of the policy that created the flow") + .def_readonly("sources", &FlowRecord::sources, "the flow's source node ids, ascending") + .def_readonly("destinations", &FlowRecord::destinations, "the flow's destination node ids, ascending") + .def_readonly("placement", &FlowRecord::placement, "the FlowPlacement (kernel) that spreads the flow over its route") + .def_readonly("admission", &FlowRecord::admission, "LOSSLESS or LOSSY") + .def_property_readonly( + "bundle", [](const FlowRecord& r) { return std::const_pointer_cast(r.bundle); }, + "the flow's route (an immutable PredDAG shared with the ledger)") + .def_property_readonly( + "route_cost", [](const FlowRecord& r) { return cost_or_none(r.route_cost); }, + "the cost of the cheapest walk of the bundle from the sources to the destinations; None when\n" + "the bundle has no such walk") + .def_property_readonly( + "offered", + [](const FlowRecord& r) -> py::object { + return r.offered.has_value() ? counter_int(*r.offered) : py::object(py::none()); + }, + "Lossy: the units offered to the flow, summed over calls (an exact int, may exceed 64 bits);\n" + "None for a Lossless flow") + .def_readonly("delivered", &FlowRecord::delivered, "the units the flow delivers to its destinations (its total)") + .def_property_readonly( + "edges", [](const FlowRecord& r) { return pairs_list(r.edges); }, + "the flow's allocation [(EdgeId, units)], positive entries in ascending EdgeId") + .def_property_readonly( + "drops", + [](const FlowRecord& r) { + py::list out; + for (const auto& [e, x] : r.drops) out.append(py::make_tuple(e, counter_int(x))); + return out; + }, + "Lossy: the units dropped per edge [(EdgeId, units)], summed over calls, positive entries in\n" + "ascending EdgeId (exact ints); empty for a Lossless flow") + .def("__repr__", [](const FlowRecord& r) { + return "FlowRecord(flow_id=(" + std::to_string(r.flow_id.first) + ", " + std::to_string(r.flow_id.second) + + "), owner=" + std::to_string(r.owner) + ", delivered=" + std::to_string(r.delivered) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/graph.cpp b/bindings/python/graph.cpp new file mode 100644 index 0000000..3db326e --- /dev/null +++ b/bindings/python/graph.cpp @@ -0,0 +1,216 @@ +// Graph: the immutable graph on integer units and its validating constructor. +#include +#include +#include + +#include +#include +#include +#include + +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The unit-value step on the capacity argument whatever its shape (the shape step comes next): +// every integer entry it holds must lie in [0, 2^62). A NumPy array is read flattened; nested +// entries of a Python sequence and a non-integer non-sequence are left to the shape step. +void check_unit_values(const py::handle& capacity_units) { + const auto out_of_domain = [](const std::string& what, const std::string& value) { + return py::value_error(what + " = " + value + " is outside the units domain [0, 2^62)"); + }; + if (py::isinstance(capacity_units)) { + py::object flat = capacity_units.attr("ravel")(); + const auto w = wide_integers(flat, "capacity_units"); + for (std::size_t i = 0; i < w.size(); ++i) { + if (!w.in(i, 0, kDomainLimit - 1)) throw out_of_domain("capacity_units[" + std::to_string(i) + "]", w.text(i)); + } + return; + } + if (!not_a_sequence(capacity_units)) { + auto seq = py::reinterpret_borrow(capacity_units); + for (std::size_t i = 0; i < seq.size(); ++i) { + py::object item = seq[i]; + if (is_nested(item)) continue; + const std::string what = "capacity_units[" + std::to_string(i) + "]"; + py::object idx = python_int(item, what); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= kDomainLimit) throw out_of_domain(what, std::string(py::str(idx))); + } + return; + } + if (!is_bool(capacity_units) && PyIndex_Check(capacity_units.ptr())) { + py::object idx = python_int(capacity_units, "capacity_units"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= kDomainLimit) throw out_of_domain("capacity_units", std::string(py::str(idx))); + } +} + +std::shared_ptr make_graph(const py::object& num_nodes, const py::object& src, + const py::object& dst, const py::object& capacity_units, + const py::object& cost, double resolution) { + // The constructor's validation order: the resolution, the unit values, the shapes, zero or + // negative costs, the remaining bounds (ids in range, then the sums); every value is read + // without narrowing, and the messages are the C++ constructor's, which re-checks the same order. + (void)resolution_exponent(resolution); + check_unit_values(capacity_units); + // The shapes: num_nodes, then every array argument rank 1 (in signature order), then equal + // lengths; no entry of src, dst or cost is read before the cost step. + const auto n = integer_value(num_nodes, 0, std::numeric_limits::max(), "num_nodes"); + const std::size_t m = sequence_length(src, "src"); + const std::size_t md = sequence_length(dst, "dst"); + const std::size_t mu = sequence_length(capacity_units, "capacity_units"); + const std::size_t mc = sequence_length(cost, "cost"); + if (md != m || mu != m || mc != m) { + throw py::value_error("src, dst, capacity_units and cost must have the same length (got " + + std::to_string(m) + ", " + std::to_string(md) + ", " + std::to_string(mu) + ", " + + std::to_string(mc) + ")"); + } + const auto cap = wide_integers(capacity_units, "capacity_units"); // rank 1, values checked + if (m > static_cast(std::numeric_limits::max())) { + throw py::value_error("the number of edges exceeds INT32_MAX (array length)"); + } + const auto w = wide_integers(cost, "cost"); + for (std::size_t i = 0; i < m; ++i) { + if (w.sign(i) < 0 || (w.sign(i) == 0 && w.value[i] < 1)) { + throw py::value_error("cost[" + std::to_string(i) + "] = " + w.text(i) + ": costs must be positive (cost >= 1)"); + } + } + const auto s = wide_integers(src, "src"); + const auto d = wide_integers(dst, "dst"); + for (std::size_t i = 0; i < m; ++i) { + if (!s.in(i, 0, n - 1) || !d.in(i, 0, n - 1)) { + throw py::value_error("edge " + std::to_string(i) + ": endpoint (" + s.text(i) + ", " + d.text(i) + + ") out of node range [0, " + std::to_string(n) + ")"); + } + } + constexpr auto kLimit = static_cast(kDomainLimit); + std::uint64_t cap_sum = 0; + std::uint64_t cost_sum = 0; + for (std::size_t i = 0; i < m; ++i) { + cap_sum += static_cast(cap.value[i]); // each below 2^62: no wrap before the check + if (cap_sum >= kLimit) { + throw py::value_error("the sum of all capacities reaches the bound of the units domain (2^62 units)"); + } + if (w.sign(i) > 0 || static_cast(w.value[i]) >= kLimit - cost_sum) { + throw py::value_error("the sum of all edge costs reaches the bound 2^62: path costs would overflow int64"); + } + cost_sum += static_cast(w.value[i]); + } + auto cap_v = cap.narrow(); + auto cost_v = w.narrow(); + auto src_v = s.narrow(); + auto dst_v = d.narrow(); + py::gil_scoped_release release; + return std::make_shared(static_cast(n), src_v, dst_v, cap_v, cost_v, resolution); +} + +} // namespace + +void register_graph(py::module_& m) { + py::class_>( + m, "Graph", + "Immutable directed multigraph on integer units (parallel edges and self-loops allowed). Edges are\n" + "given in input order and numbered by sorting on (src, cost, dst), ties kept in input order: that\n" + "number is the public EdgeId. input_index()[e] is the input position of edge e. Per-edge values\n" + "use EdgeId order; in_adj_edge_index() instead lists EdgeIds in incoming-slot order. Costs are\n" + "positive integers; capacities are int64 units of the explicit resolution q (a power of two in\n" + "[2^-32, 2^32]); physical capacities enter through to_units.") + .def(py::init(&make_graph), py::arg("num_nodes"), py::arg("src"), py::arg("dst"), + py::arg("capacity_units"), py::arg("cost"), py::arg("resolution"), + "Validates, in order, and reports the first defect as ValueError: the resolution (a power of\n" + "two in [2^-32, 2^32]), the unit values (every capacity in [0, 2^62)), the array shapes,\n" + "zero or negative costs, the remaining bounds (endpoints in range, then the\n" + "capacity and cost sums below 2^62). Integers are read without narrowing, so the error\n" + "does not depend on how the input is represented.") + .def("uid", &Graph::uid, "process-unique id (never reused)") + .def("resolution", &Graph::resolution, "the resolution q (units of capacity)") + .def("num_nodes", &Graph::num_nodes, "N: node ids are 0 .. N - 1") + .def("num_edges", &Graph::num_edges, "E: EdgeIds are 0 .. E - 1") + .def("max_cost", &Graph::max_cost, "largest edge cost (0 without edges)") + .def("cost_gcd", &Graph::cost_gcd, "gcd of the edge costs (0 without edges)") + .def( + "capacity_units", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.capacity_units(), self); + }, + "int64[E] capacities in units, id order (read-only view)") + .def( + "cost", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.cost(), self); + }, + "int64[E] costs, id order (read-only view)") + .def( + "edge_src", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.edge_src(), self); + }, + "int32[E] source node per edge (read-only view)") + .def( + "edge_dst", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.edge_dst(), self); + }, + "int32[E] destination node per edge (read-only view)") + .def( + "input_index", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.input_index(), self); + }, + "int32[E]: EdgeId -> input position (read-only view); build the inverse yourself if needed") + .def( + "row_offsets", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.row_offsets(), self); + }, + "int32[N + 1]: the out-edges of u are the ids [row_offsets[u], row_offsets[u + 1])") + .def( + "in_row_offsets", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.in_row_offsets(), self); + }, + "int32[N + 1]: the in-edges of v are in_adj_edge_index()[in_row_offsets[v] : in_row_offsets[v + 1]]") + .def( + "in_adj_edge_index", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.in_adj_edge_index(), self); + }, + "int32[E]: the in-edges of v in (cost, src, input) order, as EdgeIds") + .def( + "eligibility", + [](const Graph& g, const py::object& excluded_nodes, const py::object& excluded_edges) { + // Nodes, then edges, each range-checked before narrowing (the constructor's order). + const auto xn = wide_integers(excluded_nodes, "excluded_nodes"); + for (std::size_t i = 0; i < xn.size(); ++i) { + if (!xn.in(i, 0, static_cast(g.num_nodes()) - 1)) { + throw py::value_error("excluded node " + xn.text(i) + " out of node range [0, " + + std::to_string(g.num_nodes()) + ")"); + } + } + const auto xe = wide_integers(excluded_edges, "excluded_edges"); + for (std::size_t i = 0; i < xe.size(); ++i) { + if (!xe.in(i, 0, static_cast(g.num_edges()) - 1)) { + throw py::value_error("excluded edge " + xe.text(i) + " out of edge range [0, " + + std::to_string(g.num_edges()) + ")"); + } + } + return std::make_shared(g.eligibility(xn.narrow(), xe.narrow())); + }, + py::arg("excluded_nodes") = py::tuple(), py::arg("excluded_edges") = py::tuple(), + "Static eligibility from excluded node ids and EdgeIds (duplicates and unsorted lists\n" + "accepted, out-of-range ids rejected); an edge is allowed when it is not excluded and\n" + "both endpoints are allowed. Empty lists mean all allowed (uid 0)."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/hooks.cpp b/bindings/python/hooks.cpp new file mode 100644 index 0000000..ce81af1 --- /dev/null +++ b/bindings/python/hooks.cpp @@ -0,0 +1,236 @@ +// The private hooks (NETGRAPH_CORE_PRIVATE_HOOKS builds only): the test controls +// of the creation cap and of Python and native allocation failures, and the apportionment +// kernels. +#include +#include +#include + +#include "registration.hpp" + +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/kernels.hpp" +#include "netgraph/core/detail/private_hooks.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/types.hpp" +#include "integers.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// Take the normalized exception, including its traceback, without replacing an existing +// exception instance. PyErr_GetRaisedException was added in Python 3.12. +py::object take_raised_exception() { +#if PY_VERSION_HEX >= 0x030C0000 + return py::reinterpret_steal(PyErr_GetRaisedException()); +#else + PyObject* type = nullptr; + PyObject* value = nullptr; + PyObject* traceback = nullptr; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + const auto kind = py::reinterpret_steal(type); + auto error = py::reinterpret_steal(value); + const auto trace = py::reinterpret_steal(traceback); + if (trace && PyException_SetTraceback(error.ptr(), trace.ptr()) < 0) throw py::error_already_set(); + return error; +#endif +} + +// The allocation-failure hook: the k-th object-domain allocation fails (Python's allocator +// wrapped for one call, then restored). The allocator is process-wide while the call runs, so +// the hook serves one caller at a time (the tests); its counters are atomics because the module +// does not hold the GIL for every thread (mod_gil_not_used). +PyMemAllocatorEx g_saved_allocator {}; +std::atomic g_fail_countdown {-1}; +std::atomic g_fail_seen {0}; +std::atomic g_fail_hit {false}; + +bool fail_now() { + g_fail_seen.fetch_add(1); + if (g_fail_countdown.load() < 0) return false; + if (g_fail_countdown.fetch_sub(1) == 0) { + g_fail_hit.store(true); + return true; + } + return false; +} +void* failing_malloc(void*, std::size_t n) { return fail_now() ? nullptr : g_saved_allocator.malloc(g_saved_allocator.ctx, n); } +void* failing_calloc(void*, std::size_t n, std::size_t size) { + return fail_now() ? nullptr : g_saved_allocator.calloc(g_saved_allocator.ctx, n, size); +} +void* failing_realloc(void*, void* ptr, std::size_t n) { + return fail_now() ? nullptr : g_saved_allocator.realloc(g_saved_allocator.ctx, ptr, n); +} +void forwarding_free(void*, void* ptr) { g_saved_allocator.free(g_saved_allocator.ctx, ptr); } + +// The native allocation-failure hook: while armed for one call, the k-th allocation of +// the calling thread through the extension's global operator new (the bindings and the library +// linked into the extension) throws std::bad_alloc, and the thread's allocations and deletions +// are counted, so a test also sees what a failed call left allocated. Other threads and other +// images (libc++, Python) are not counted; the aligned forms are not replaced. The counters are +// atomics for the same reason as above. +std::atomic g_native_armed {false}; +std::atomic g_native_owner {}; +std::atomic g_native_countdown {-1}; +std::atomic g_native_seen {0}; +std::atomic g_native_live {0}; +std::atomic g_native_hit {false}; + +bool native_counted() noexcept { + return g_native_armed.load(std::memory_order_relaxed) && g_native_owner.load() == std::this_thread::get_id(); +} + +void* native_allocate(std::size_t n) { + if (native_counted()) { + g_native_seen.fetch_add(1); + if (g_native_countdown.load() >= 0 && g_native_countdown.fetch_sub(1) == 0) { + g_native_hit.store(true); + throw std::bad_alloc(); + } + } + void* p = std::malloc(n == 0 ? 1 : n); + if (p == nullptr) throw std::bad_alloc(); + if (native_counted()) g_native_live.fetch_add(1); + return p; +} + +void* native_allocate_nothrow(std::size_t n) noexcept { + try { + return native_allocate(n); + } catch (...) { + return nullptr; + } +} + +void native_free(void* p) noexcept { + if (p != nullptr && native_counted()) g_native_live.fetch_sub(1); + std::free(p); +} + +} // namespace + +void register_private_hooks(py::module_& m) { + m.def( + "_max_flows_per_call", + [](const py::object& value) { + auto& cap = detail::max_flows_per_call(); + const auto old = cap.load(); + if (!value.is_none()) cap.store(integer_value(value, 0, kMaxFlowsPerCall, "max_flows_per_call")); + return old; + }, + py::arg("value") = py::none(), + "private test hook: the call-wide creation cap (MAX_FLOWS_PER_CALL by default); returns the old value"); + + m.def( + "_fail_python_allocation", + [](const py::object& fn, const py::tuple& args, long k) { + // Calls fn(*args) with the k-th Python object allocation (from 0) failing, then + // restores the allocator. Returns (hit, allocations seen, result or None, exception or + // None). One caller at a time: the allocator is process-wide while the call runs. + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &g_saved_allocator); + PyMemAllocatorEx hook {nullptr, failing_malloc, failing_calloc, failing_realloc, forwarding_free}; + g_fail_countdown.store(k); + g_fail_seen.store(0); + g_fail_hit.store(false); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &hook); + PyObject* r = PyObject_CallObject(fn.ptr(), args.ptr()); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &g_saved_allocator); + g_fail_countdown.store(-1); + py::object result = r != nullptr ? py::reinterpret_steal(r) : py::object(py::none()); + py::object error = py::none(); + if (r == nullptr) error = take_raised_exception(); + return py::make_tuple(g_fail_hit.load(), g_fail_seen.load(), result, error); + }, + py::arg("fn"), py::arg("args"), py::arg("k"), + "private test hook: fn(*args) with the k-th Python object allocation failing -> (hit, seen, result, error)"); + + m.def( + "_fail_native_allocation", + [](const py::object& fn, const py::tuple& args, long k) { + // Calls fn(*args) with the calling thread's k-th native allocation (from 0) failing. + // Returns (hit, allocations seen, result or None, exception or None, allocations of the + // call still live). One caller at a time. + g_native_countdown.store(k); + g_native_seen.store(0); + g_native_live.store(0); + g_native_hit.store(false); + g_native_owner.store(std::this_thread::get_id()); + g_native_armed.store(true); + PyObject* r = PyObject_CallObject(fn.ptr(), args.ptr()); + g_native_armed.store(false); + g_native_countdown.store(-1); + py::object result = r != nullptr ? py::reinterpret_steal(r) : py::object(py::none()); + py::object error = py::none(); + if (r == nullptr) error = take_raised_exception(); + return py::make_tuple(g_native_hit.load(), g_native_seen.load(), result, error, g_native_live.load()); + }, + py::arg("fn"), py::arg("args"), py::arg("k"), + "private test hook: fn(*args) with the k-th native allocation failing -> (hit, seen, result, error, live)"); + + m.def( + "_round_robin", + [](const py::object& total, const py::object& k) { + const auto t = integer_value(total, 0, kDomainLimit - 1, "total"); + const auto n = integer_value(k, 1, 1 << 20, "k"); + std::vector out(static_cast(n)); + detail::round_robin(t, out); + return out; + }, + py::arg("total"), py::arg("k"), + "private: round robin of total units over k members (the first total % k get one unit more)"); + + m.def( + "_highest_averages", + [](const py::object& total, const py::object& weights) { + const auto t = integer_value(total, 0, kDomainLimit - 1, "total"); + const auto w = wide_integers(weights, "weights"); + if (w.size() == 0) throw py::value_error("highest averages over no member"); + std::uint64_t sum = 0; + for (std::size_t i = 0; i < w.size(); ++i) { + if (!w.in(i, 1, kDomainLimit - 1)) { + throw py::value_error("weights[" + std::to_string(i) + "] = " + w.text(i) + " is outside [1, 2^62)"); + } + sum += static_cast(w.value[i]); + if (sum >= static_cast(kDomainLimit)) throw py::value_error("the weights sum to 2^62 or more"); + } + const auto wv = w.narrow(); + std::vector out(wv.size()); + detail::highest_averages(t, wv, out); + return out; + }, + py::arg("total"), py::arg("weights"), + "private: highest-averages (D'Hondt) apportionment of total units over positive weights, ties by\n" + "the lower index"); +} + +} // namespace netgraph::core::bindings + +// The extension's global allocation functions in a hooks build: the native allocation-failure +// hook above. Unarmed they allocate as the defaults do (malloc and free, which every image's +// operator delete accepts). +using netgraph::core::bindings::native_allocate; +using netgraph::core::bindings::native_allocate_nothrow; +using netgraph::core::bindings::native_free; + +void* operator new(std::size_t n) { return native_allocate(n); } +void* operator new[](std::size_t n) { return native_allocate(n); } +void* operator new(std::size_t n, const std::nothrow_t&) noexcept { return native_allocate_nothrow(n); } +void* operator new[](std::size_t n, const std::nothrow_t&) noexcept { return native_allocate_nothrow(n); } +void operator delete(void* p) noexcept { native_free(p); } +void operator delete[](void* p) noexcept { native_free(p); } +void operator delete(void* p, std::size_t) noexcept { native_free(p); } +void operator delete[](void* p, std::size_t) noexcept { native_free(p); } +void operator delete(void* p, const std::nothrow_t&) noexcept { native_free(p); } +void operator delete[](void* p, const std::nothrow_t&) noexcept { native_free(p); } + +#endif diff --git a/bindings/python/integers.hpp b/bindings/python/integers.hpp new file mode 100644 index 0000000..d5b7a21 --- /dev/null +++ b/bindings/python/integers.hpp @@ -0,0 +1,232 @@ +/* + Integer arguments from Python: Python ints, NumPy integer scalars and integer arrays, read + without narrowing (WideInts) and validated in the C++ constructors' order before any + conversion to the C++ types, so the reported error does not depend on how the input is + represented; bools and floats are rejected, never truncated. Also the shape step of sequence + arguments, the per-edge amount arrays and the node sets. +*/ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/types.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +inline bool is_bool(const py::handle& obj) { + return PyBool_Check(obj.ptr()) || py::isinstance(obj, py::module_::import("numpy").attr("bool_")); +} + +// The Python int of an integer-like object (a Python int or a NumPy integer scalar); bools and +// everything else are rejected, and an exception of the object's own __index__ propagates as +// raised. The result is an exact int, so PyLong_AsLongLongAndOverflow cannot fail on it. +inline py::object python_int(const py::handle& item, const std::string& what) { + if (is_bool(item) || !PyIndex_Check(item.ptr())) { + throw py::value_error(what + " must be an integer (got " + + std::string(py::str(py::type::of(item).attr("__name__"))) + ")"); + } + py::object idx = py::reinterpret_steal(PyNumber_Index(item.ptr())); + if (!idx) throw py::error_already_set(); + return idx; +} + +// One integer of [lo, hi] (scalar arguments). +inline std::int64_t integer_value(const py::handle& item, std::int64_t lo, std::int64_t hi, const std::string& what) { + py::object idx = python_int(item, what); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < lo || v > hi) { + throw py::value_error(what + " = " + std::string(py::str(idx)) + " is out of range [" + + std::to_string(lo) + ", " + std::to_string(hi) + "]"); + } + return static_cast(v); +} + +// A 1-D integer sequence read without narrowing: every entry's int64 value where it fits, and +// the sign of the overflow where it does not (+1 above INT64_MAX, -1 below INT64_MIN). Only type +// defects raise here (a non-integer, a bool, a float array, a non-sequence); every value bound +// is checked by the caller in the C++ constructor's order, so the reported error does not depend +// on the representation of the input. +struct WideInts { + py::object source; + std::vector value; + std::vector overflow; // empty when no entry overflows + [[nodiscard]] std::size_t size() const noexcept { return value.size(); } + [[nodiscard]] int sign(std::size_t i) const noexcept { return overflow.empty() ? 0 : overflow[i]; } + // The entry as Python prints it (exact also beyond int64). + [[nodiscard]] std::string text(std::size_t i) const { + if (sign(i) == 0) return std::to_string(value[i]); + return std::string(py::str(source[py::int_(i)])); + } + [[nodiscard]] bool in(std::size_t i, std::int64_t lo, std::int64_t hi) const noexcept { + return sign(i) == 0 && value[i] >= lo && value[i] <= hi; + } + // The entry as an exact Python int (an overflowing entry compared with another). The entry is + // converted again here, so its __index__ may now fail: that exception propagates as raised. + [[nodiscard]] py::object exact(std::size_t i) const { + py::object item = source[py::int_(i)]; + py::object idx = py::reinterpret_steal(PyNumber_Index(item.ptr())); + if (!idx) throw py::error_already_set(); + return idx; + } + template + [[nodiscard]] std::vector narrow() const { // after every bound was checked + return std::vector(value.begin(), value.end()); + } +}; + +inline bool not_a_sequence(const py::handle& obj) { + return py::isinstance(obj) || py::isinstance(obj) || !py::isinstance(obj); +} + +inline WideInts wide_integers(const py::handle& obj, const char* name) { + WideInts out; + out.source = py::reinterpret_borrow(obj); + if (py::isinstance(obj)) { + auto arr = py::reinterpret_borrow(obj); + if (arr.ndim() != 1) { + throw py::value_error(std::string(name) + " must be a 1-D array (got " + std::to_string(arr.ndim()) + + " dimensions)"); + } + const char kind = arr.dtype().kind(); + if (kind != 'i' && kind != 'u') { + throw py::value_error(std::string(name) + " entries must be integers (got dtype " + + std::string(py::str(arr.dtype())) + ")"); + } + out.value.reserve(static_cast(arr.size())); + if (kind == 'u' && arr.itemsize() == 8) { + auto u = py::array_t::ensure(arr); + auto r = u.unchecked<1>(); + constexpr auto kMax = static_cast(std::numeric_limits::max()); + for (py::ssize_t i = 0; i < r.shape(0); ++i) { + if (r(i) > kMax) { + if (out.overflow.empty()) out.overflow.assign(static_cast(r.shape(0)), 0); + out.overflow[static_cast(i)] = 1; + out.value.push_back(std::numeric_limits::max()); + } else { + out.value.push_back(static_cast(r(i))); + } + } + return out; + } + auto a = py::array_t::ensure(arr); + auto r = a.unchecked<1>(); + for (py::ssize_t i = 0; i < r.shape(0); ++i) out.value.push_back(r(i)); + return out; + } + if (not_a_sequence(obj)) throw py::value_error(std::string(name) + " must be a sequence of integers"); + auto seq = py::reinterpret_borrow(obj); + const std::size_t n = seq.size(); + out.value.reserve(n); + for (std::size_t i = 0; i < n; ++i) { + py::object idx = python_int(seq[i], std::string(name) + "[" + std::to_string(i) + "]"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0) { + if (out.overflow.empty()) out.overflow.assign(n, 0); + out.overflow[i] = static_cast(overflow); + out.value.push_back(overflow > 0 ? std::numeric_limits::max() + : std::numeric_limits::min()); + } else { + out.value.push_back(static_cast(v)); + } + } + return out; +} + +// An entry that is itself a sequence or an array of rank >= 1 (a shape defect of its argument). +inline bool is_nested(const py::handle& item) { + if (py::isinstance(item)) return py::reinterpret_borrow(item).ndim() >= 1; + return !py::isinstance(item) && !py::isinstance(item) && PySequence_Check(item.ptr()) != 0; +} + +// The shape step: the length of a rank-1 integer sequence argument. A NumPy array of another +// rank (0-D included), a nested sequence or a non-sequence is a shape defect. Entry types and +// values are not read here; their steps come later. +inline std::size_t sequence_length(const py::handle& obj, const char* name) { + if (py::isinstance(obj)) { + auto arr = py::reinterpret_borrow(obj); + if (arr.ndim() != 1) { + throw py::value_error(std::string(name) + " must be a 1-D array (got " + std::to_string(arr.ndim()) + + " dimensions)"); + } + return static_cast(arr.size()); + } + if (not_a_sequence(obj)) { + throw py::value_error(std::string(name) + " must be a 1-D sequence of integers (got " + + std::string(py::str(py::type::of(obj).attr("__name__"))) + ")"); + } + auto seq = py::reinterpret_borrow(obj); + const std::size_t n = seq.size(); + for (std::size_t i = 0; i < n; ++i) { + if (is_nested(seq[i])) { + throw py::value_error(std::string(name) + " must be a 1-D sequence of integers (entry " + std::to_string(i) + + " is a sequence)"); + } + } + return n; +} + +// A per-edge integer array in [0, capacity_units[e]] (a ledger baseline, a search residual, +// the capacities `view` of _place_global_units), validated on the unnarrowed values: the length +// (shape step), then every entry in EdgeId order. `label` names the entries in a type error +// ("residual (units)", "view (capacity units)"). The checked values come back unnarrowed: a +// caller narrows them later (_place_global_units, inside its call). +inline WideInts edge_amounts(const py::object& obj, const Graph& g, const char* name, const char* label) { + const auto num_edges = static_cast(g.num_edges()); + const std::size_t len = sequence_length(obj, name); + if (len != num_edges) { + throw py::value_error(std::string(name) + " has length " + std::to_string(len) + ", expected " + + std::to_string(num_edges) + " (one entry per edge)"); + } + auto v = wide_integers(obj, label); + const auto capacity = g.capacity_units(); + for (std::size_t e = 0; e < num_edges; ++e) { + if (!v.in(e, 0, capacity[e])) { + throw py::value_error(std::string(name) + "[" + std::to_string(e) + "] = " + v.text(e) + " is outside [0, capacity_units[" + + std::to_string(e) + "] = " + std::to_string(capacity[e]) + "]"); + } + } + return v; +} + +// A per-edge integer array in [0, capacity_units[e]] (a ledger baseline, a search residual), +// validated on the unnarrowed values: the length (shape step), then every entry. +inline std::vector edge_units(const py::object& obj, const Graph& g, const char* name) { + return edge_amounts(obj, g, name, (std::string(name) + " (units)").c_str()).narrow(); +} + +// A node set's ids, non-empty and range-checked before narrowing (detail::node_set's messages). +inline std::vector checked_ids(const WideInts& ids, std::int32_t num_nodes, const char* name) { + if (ids.size() == 0) throw py::value_error(detail::empty_set_error(name)); + for (std::size_t i = 0; i < ids.size(); ++i) { + if (!ids.in(i, 0, static_cast(num_nodes) - 1)) { + throw py::value_error(detail::node_range_error(name, ids.text(i), num_nodes)); + } + } + return ids.narrow(); +} + +// A node set: one NodeId or a sequence of them. +inline std::vector node_ids(const py::object& obj, std::int32_t num_nodes, const char* name) { + if (!is_bool(obj) && PyIndex_Check(obj.ptr()) && !py::isinstance(obj)) { + py::list one; + one.append(obj); + return checked_ids(wide_integers(one, name), num_nodes, name); + } + return checked_ids(wide_integers(obj, name), num_nodes, name); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/module.cpp b/bindings/python/module.cpp index e01488f..394f3a5 100644 --- a/bindings/python/module.cpp +++ b/bindings/python/module.cpp @@ -1,686 +1,65 @@ /* - Pybind11 module exposing NetGraph-Core C++ APIs to Python. - - Uses NumPy arrays with zero-copy spans where possible for FlowState/FlowGraph views (read-only) - - Returns distances as float64 arrays by default (inf for unreachable); dtype='int64' available - - Validates edge/node masks for bool dtype and length - - StrictMultiDiGraph view methods return copies (by design) to avoid exposing lifetime + Pybind11 module `_netgraph_core`: the graph on integer units with Eligibility and the unit + conversions, the route structures (PredDAG, Path), the selection and placement axes, the + ledger with its records, the policy engine with its configuration and results, the DagCache, + the evaluators (capacity, batch_capacity, sensitivity) and the public search (spf, ksp, + resolve_to_paths); and one private entry: the flow core's _place_global_units. + A build with the private hooks (CMake NETGRAPH_CORE_PRIVATE_HOOKS: the editable development + build) adds the test controls and the parity diagnostics. + + - Graph accessors return read-only NumPy views that keep the graph alive (no per-call copy). + - Integer inputs (capacities, costs, ids, views, caps) must be integers: Python ints, NumPy + integer scalars or integer arrays; floats and bools are rejected with ValueError, never + truncated. Values are read without narrowing (WideInts) and validated in the C++ + constructors' order before any conversion to the C++ types. Physical values + enter only through to_units; the resolution is validated before anything else. + - std::invalid_argument from Core maps to ValueError; enumeration values outside their + enumeration are rejected at construction (ValueError). + - _place_global_units is a private entry point for the flow checks: it returns the flow + core's payload as a dict of Python ints and lists. + - Amounts and counters cross as Python ints (exact, also beyond 64 bits); per-edge arrays as + int64 NumPy arrays in EdgeId order; +inf is accepted only as a request (float('inf')). + - Python spells constants in UPPER_SNAKE_CASE: MAX_FLOWS_PER_CALL is the C++ kMaxFlowsPerCall. */ #include -#include -#include -#include -#include -#include -#include "netgraph/core/k_shortest_paths.hpp" -#include "netgraph/core/max_flow.hpp" -#include "netgraph/core/flow_state.hpp" -#include "netgraph/core/shortest_paths.hpp" -#include "netgraph/core/strict_multidigraph.hpp" -#include "netgraph/core/types.hpp" -#include "netgraph/core/flow_graph.hpp" -#include "netgraph/core/flow.hpp" -#include "netgraph/core/flow_policy.hpp" -#include "netgraph/core/backend.hpp" -#include "netgraph/core/algorithms.hpp" -#include "netgraph/core/options.hpp" -#include "netgraph/core/profiling.hpp" +#include "registration.hpp" namespace py = pybind11; using namespace netgraph::core; -// Helper opaque wrappers (defined at file scope to ensure stable type identity) -struct PyBackend { BackendPtr impl; }; -struct PyGraph { GraphHandle handle; py::object keep_alive; std::int32_t num_nodes; std::int32_t num_edges; }; - -// Helpers to check NumPy arrays -namespace { -// Small helper: copy a span into a new NumPy array of the same dtype. -template -py::array_t copy_to_numpy(std::span s) { - py::array_t arr(s.size()); - if (!s.empty()) std::memcpy(arr.mutable_data(), s.data(), s.size()*sizeof(T)); - return arr; -} - -// Owns a bool[] buffer and exposes a const span view onto it. -struct BoolSpan { - std::unique_ptr buf; - std::span view; -}; - -// Validate 1-D C-contig NumPy bool array and deep-copy into bool[]. -static BoolSpan to_bool_span_from_numpy(py::object obj, std::size_t expected_len, const char* what) { - BoolSpan out; - if (obj.is_none()) return out; // empty span - py::array arr = obj.cast(); - if (!(arr.flags() & py::array::c_style)) { - throw py::type_error(std::string(what) + " must be C-contiguous"); - } - auto b = arr.request(); - if (b.ndim != 1 || b.format != py::format_descriptor::format()) { - throw py::type_error(std::string(what) + " must be 1-D bool"); - } - if (static_cast(b.shape[0]) != expected_len) { - throw py::type_error(std::string(what) + " length must equal " + std::to_string(expected_len)); - } - out.buf.reset(new bool[expected_len]); - // NumPy bool_ uses 1 byte; C++ bool is 1 byte on supported platforms. - std::memcpy(out.buf.get(), b.ptr, expected_len * sizeof(bool)); - out.view = std::span(out.buf.get(), expected_len); - return out; -} -} // namespace - -// Helpers to check NumPy arrays -template -static std::span as_span(const py::array& arr, const char* name) { - if (!py::isinstance>(arr)) { - throw py::type_error(std::string(name) + ": expected numpy array of correct dtype"); - } - if (!(arr.flags() & py::array::c_style)) { - throw py::type_error(std::string(name) + ": array must be C-contiguous (use np.ascontiguousarray)"); - } - auto buf = arr.request(); - if (buf.ndim != 1) { - throw py::type_error(std::string(name) + ": expected 1-D array"); - } - return std::span(static_cast(buf.ptr), static_cast(buf.size)); -} - PYBIND11_MODULE(_netgraph_core, m, py::mod_gil_not_used()) { - m.doc() = "NetGraph-Core C++ bindings"; - - py::enum_(m, "EdgeTieBreak") - .value("DETERMINISTIC", EdgeTieBreak::Deterministic) - .value("PREFER_HIGHER_RESIDUAL", EdgeTieBreak::PreferHigherResidual); - - py::class_(m, "EdgeSelection") - .def(py::init<>()) - .def(py::init([](bool multi_edge, bool require_capacity, EdgeTieBreak tie_break){ - EdgeSelection s; s.multi_edge = multi_edge; s.require_capacity = require_capacity; s.tie_break = tie_break; return s; - }), - py::kw_only(), - py::arg("multi_edge") = true, - py::arg("require_capacity") = false, - py::arg("tie_break") = EdgeTieBreak::Deterministic) - .def_readwrite("multi_edge", &EdgeSelection::multi_edge) - .def_readwrite("require_capacity", &EdgeSelection::require_capacity) - .def_readwrite("tie_break", &EdgeSelection::tie_break); - - py::enum_(m, "FlowPlacement") - .value("PROPORTIONAL", FlowPlacement::Proportional) - .value("EQUAL_BALANCED", FlowPlacement::EqualBalanced) - .value("EQUAL_BALANCED_LOSSY", FlowPlacement::EqualBalancedLossy); - - py::class_(m, "StrictMultiDiGraph") - .def_static( - "from_arrays", - [](std::int32_t num_nodes, - py::array src, py::array dst, - py::array capacity, py::array cost, - py::array ext_edge_ids) { - // public API: src/dst are int32; pass through as int32 to core - auto src_s = as_span(src, "src"); - auto dst_s = as_span(dst, "dst"); - if (src_s.size() != dst_s.size()) throw py::type_error("src and dst must have the same length"); - auto cap_s = as_span(capacity, "capacity"); - // Cost dtype must be int64 to match internal Cost type - auto cost_s = as_span(cost, "cost"); - // ext_edge_ids required: maps internal edge indices to caller's stable IDs - auto ext_s = as_span(ext_edge_ids, "ext_edge_ids"); - return StrictMultiDiGraph::from_arrays(num_nodes, - src_s, - dst_s, - cap_s, cost_s, ext_s); - }, - py::arg("num_nodes"), py::arg("src"), py::arg("dst"), py::arg("capacity"), py::arg("cost"), - py::arg("ext_edge_ids")) - .def("num_nodes", &StrictMultiDiGraph::num_nodes) - .def("num_edges", &StrictMultiDiGraph::num_edges) - .def("capacity_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.capacity_view()); }) - .def("edge_src_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.edge_src_view()); }) - .def("edge_dst_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.edge_dst_view()); }) - .def("ext_edge_ids_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.ext_edge_ids_view()); }) - .def("cost_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.cost_view()); }) - .def("row_offsets_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.row_offsets_view()); }) - .def("col_indices_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.col_indices_view()); }) - .def("adj_edge_index_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.adj_edge_index_view()); }) - .def("in_row_offsets_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.in_row_offsets_view()); }) - .def("in_col_indices_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.in_col_indices_view()); }) - .def("in_adj_edge_index_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.in_adj_edge_index_view()); }); - - // Backend and Algorithms - - // Opaque wrapper types - py::class_(m, "Backend") - .def_static("cpu", [](){ return PyBackend{ make_cpu_backend() }; }); - - py::class_(m, "Graph") - // Opaque holder; constructed via Algorithms.build_graph - .def_property_readonly("num_nodes", [](const PyGraph& pg){ return pg.num_nodes; }) - .def_property_readonly("num_edges", [](const PyGraph& pg){ return pg.num_edges; }); - - py::class_>(m, "Algorithms") - .def(py::init([](const PyBackend& b){ return std::make_shared(b.impl); }), py::arg("backend")) - .def("build_graph", [](const Algorithms& algs, py::object graph_obj){ - // graph_obj is py::object so PyGraph can keep it alive; check the type - // explicitly, since a bare .cast<> failure surfaces as an opaque - // "RuntimeError: Unable to cast ... to C++ type '?'". - if (!py::isinstance(graph_obj)) { - throw py::type_error("build_graph: graph must be a StrictMultiDiGraph"); - } - const StrictMultiDiGraph& g = graph_obj.cast(); - auto gh = algs.build_graph(g); - // Keep the Python graph object alive alongside the handle when - // referencing a non-owned graph instance. - return PyGraph{ gh, graph_obj, g.num_nodes(), g.num_edges() }; - }, py::arg("graph")) - .def("spf", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, - py::object dst, py::object selection_obj, py::object residual_obj, - py::object node_mask, py::object edge_mask, bool multipath, std::string dtype) -> py::tuple { - if (src < 0 || src >= pg.num_nodes) throw py::value_error("src out of range"); - SpfOptions opts; if (!selection_obj.is_none()) opts.selection = py::cast(selection_obj); - if (!dst.is_none()) { auto d = py::cast(dst); if (d < 0 || d >= pg.num_nodes) throw py::value_error("dst out of range"); opts.dst = static_cast(d); } - std::vector residual_vec; - if (!residual_obj.is_none()) { - auto arr = py::cast(residual_obj); - if (!(arr.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous (np.ascontiguousarray)"); - auto buf = arr.request(); - if (buf.ndim != 1 || buf.format != py::format_descriptor::format()) throw py::type_error("residual must be 1-D float64"); - // Check length here so this matches the TypeError raised by every other - // length check (masks, FlowState residual); otherwise the core throws - // std::invalid_argument, which reaches Python as ValueError. - if (static_cast(buf.shape[0]) != pg.num_edges) { - throw py::type_error("residual length must equal " + std::to_string(pg.num_edges)); - } - residual_vec.resize(static_cast(buf.shape[0])); - std::memcpy(residual_vec.data(), buf.ptr, residual_vec.size()*sizeof(double)); - opts.residual = std::span(residual_vec.data(), residual_vec.size()); - } - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - opts.node_mask = node_bs.view; - opts.edge_mask = edge_bs.view; - opts.multipath = multipath; - py::gil_scoped_release rel; auto res = algs.spf(pg.handle, src, opts); py::gil_scoped_acquire acq; - auto maxc = std::numeric_limits::max(); - if (dtype == "int64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i(maxc) : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else if (dtype == "float64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i::infinity() : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else { - throw py::value_error("dtype must be 'float64' or 'int64'"); - } - }, py::arg("graph"), py::arg("src"), py::arg("dst") = py::none(), py::kw_only(), py::arg("selection") = py::none(), py::arg("residual") = py::none(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none(), py::arg("multipath") = true, py::arg("dtype") = "float64") - .def("spf_to", [](const Algorithms& algs, const PyGraph& pg, std::int32_t dst, - py::object selection_obj, py::object residual_obj, - py::object node_mask, py::object edge_mask, bool multipath, - py::object fanout_obj, std::string dtype) -> py::tuple { - if (dst < 0 || dst >= pg.num_nodes) throw py::value_error("dst out of range"); - SpfToOptions opts; if (!selection_obj.is_none()) opts.selection = py::cast(selection_obj); - std::vector residual_vec; - if (!residual_obj.is_none()) { - auto arr = py::cast(residual_obj); - if (!(arr.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous (np.ascontiguousarray)"); - auto buf = arr.request(); - if (buf.ndim != 1 || buf.format != py::format_descriptor::format()) throw py::type_error("residual must be 1-D float64"); - if (static_cast(buf.shape[0]) != pg.num_edges) { - throw py::type_error("residual length must equal " + std::to_string(pg.num_edges)); - } - residual_vec.resize(static_cast(buf.shape[0])); - std::memcpy(residual_vec.data(), buf.ptr, residual_vec.size()*sizeof(double)); - opts.residual = std::span(residual_vec.data(), residual_vec.size()); - } - std::vector fanout_vec; - if (!fanout_obj.is_none()) { - for (auto item : py::iterable(fanout_obj)) { - auto e = py::cast(item); - if (e < 0 || e >= pg.num_edges) throw py::value_error("fanout edge id out of range"); - fanout_vec.push_back(static_cast(e)); - } - opts.fanout_edges = std::span(fanout_vec.data(), fanout_vec.size()); - } - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - opts.node_mask = node_bs.view; - opts.edge_mask = edge_bs.view; - opts.multipath = multipath; - py::gil_scoped_release rel; auto res = algs.spf_to(pg.handle, dst, opts); py::gil_scoped_acquire acq; - auto maxc = std::numeric_limits::max(); - if (dtype == "int64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i(maxc) : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else if (dtype == "float64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i::infinity() : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else { - throw py::value_error("dtype must be 'float64' or 'int64'"); - } - }, py::arg("graph"), py::arg("dst"), py::kw_only(), py::arg("selection") = py::none(), py::arg("residual") = py::none(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none(), py::arg("multipath") = true, py::arg("fanout_edges") = py::none(), py::arg("dtype") = "float64", - "Shortest paths from every node to dst (reverse SPF). Returns (distances_to_dst, dag); the DAG is forward-oriented and valid for placement from any node toward dst. fanout_edges are added to the DAG regardless of cost (see shortest_paths.hpp).") - .def("ksp", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, std::int32_t dst, - int k, py::object max_cost_factor, bool unique, py::object node_mask, py::object edge_mask, std::string dtype){ - if (src < 0 || src >= pg.num_nodes || dst < 0 || dst >= pg.num_nodes) throw py::value_error("src/dst out of range"); - if (k <= 0) throw py::value_error("k must be >= 1"); - if (dtype != "float64" && dtype != "int64") throw py::value_error("dtype must be 'float64' or 'int64'"); - KspOptions opts; opts.k = k; opts.unique = unique; if (!max_cost_factor.is_none()) opts.max_cost_factor = py::cast(max_cost_factor); - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - opts.node_mask = node_bs.view; opts.edge_mask = edge_bs.view; - py::gil_scoped_release rel; auto items = algs.ksp(pg.handle, src, dst, opts); py::gil_scoped_acquire acq; - py::list out; - auto maxc = std::numeric_limits::max(); - for (auto& pr : items) { - auto& dist = pr.first; - if (dtype == "int64") { - py::array_t dist_arr(dist.size()); - auto* outp = dist_arr.mutable_data(); - for (std::size_t i=0;i(maxc) : static_cast(dist[i]); - out.append(py::make_tuple(std::move(dist_arr), pr.second)); - } else if (dtype == "float64") { - py::array_t dist_arr(dist.size()); - auto* outp = dist_arr.mutable_data(); - for (std::size_t i=0;i::infinity() : static_cast(dist[i]); - out.append(py::make_tuple(std::move(dist_arr), pr.second)); - } - } - return out; - }, py::arg("graph"), py::arg("src"), py::arg("dst"), py::kw_only(), py::arg("k"), py::arg("max_cost_factor") = py::none(), py::arg("unique") = true, py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none(), py::arg("dtype") = "float64") - .def("max_flow", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, std::int32_t dst, - FlowPlacement placement, bool shortest_path, bool require_capacity, bool with_edge_flows, bool with_reachable, bool with_residuals, - py::object node_mask, py::object edge_mask){ - if (src < 0 || src >= pg.num_nodes || dst < 0 || dst >= pg.num_nodes) throw py::value_error("src/dst out of range"); - MaxFlowOptions o; o.placement = placement; o.shortest_path = shortest_path; o.require_capacity = require_capacity; o.with_edge_flows = with_edge_flows; o.with_reachable = with_reachable; o.with_residuals = with_residuals; - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - o.node_mask = node_bs.view; o.edge_mask = edge_bs.view; - py::gil_scoped_release rel; auto res = algs.max_flow(pg.handle, src, dst, o); py::gil_scoped_acquire acq; return res; - }, py::arg("graph"), py::arg("src"), py::arg("dst"), py::kw_only(), py::arg("flow_placement") = FlowPlacement::Proportional, py::arg("shortest_path") = false, py::arg("require_capacity") = true, py::arg("with_edge_flows") = false, py::arg("with_reachable") = false, py::arg("with_residuals") = false, py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()) - .def("batch_max_flow", [](const Algorithms& algs, const PyGraph& pg, py::array pairs, - py::object node_masks, py::object edge_masks, - FlowPlacement placement, bool shortest_path, bool require_capacity, - bool with_edge_flows, bool with_reachable, bool with_residuals){ - // Check the dtype, not the buffer format string: NumPy spells int32 as - // NPY_LONG ('l') on LLP64 (Windows) and NPY_INT ('i') on LP64, while - // format_descriptor is always 'i'. Comparing format strings - // therefore rejects a genuine int32 array on Windows. isinstance uses - // dtype equivalence, matching as_span() above. - if (!py::isinstance>(pairs)) throw py::type_error("pairs dtype must be int32"); - if (!(pairs.flags() & py::array::c_style)) throw py::type_error("pairs must be C-contiguous (use np.ascontiguousarray)"); - auto buf = pairs.request(); - if (buf.ndim != 2 || buf.shape[1] != 2) throw py::type_error("pairs must be shape [B,2]"); - const std::size_t B = static_cast(buf.shape[0]); - std::vector> pp; pp.reserve(B); - auto* p = static_cast(buf.ptr); - for (std::size_t i=0;i= pg.num_nodes || - pp[i].second < 0 || pp[i].second >= pg.num_nodes) { - throw py::value_error("pairs[" + std::to_string(i) + "] has a src/dst out of range"); - } - } - std::vector node_bufs, edge_bufs; - std::vector> node_spans; std::vector> edge_spans; - auto parse_mask_list = [&](py::object list_obj, std::size_t expected_len, const char* what, - std::vector& bufs, std::vector>& spans) { - if (list_obj.is_none()) return; - auto seq = py::cast(list_obj); - if (static_cast(py::len(seq)) != B) - throw py::type_error(std::string(what) + " length must equal number of pairs"); - bufs.reserve(B); spans.reserve(B); - for (auto item : seq) { - auto bs = to_bool_span_from_numpy(item, expected_len, what); - spans.push_back(bs.view); - bufs.push_back(std::move(bs)); // keep storage alive across GIL release - } - }; - parse_mask_list(node_masks, static_cast(pg.num_nodes), "node_masks", node_bufs, node_spans); - parse_mask_list(edge_masks, static_cast(pg.num_edges), "edge_masks", edge_bufs, edge_spans); - MaxFlowOptions o; o.placement = placement; o.shortest_path = shortest_path; o.require_capacity = require_capacity; o.with_edge_flows = with_edge_flows; o.with_reachable = with_reachable; o.with_residuals = with_residuals; - py::gil_scoped_release rel; auto out = algs.batch_max_flow(pg.handle, pp, o, node_spans, edge_spans); py::gil_scoped_acquire acq; return out; - }, py::arg("graph"), py::arg("pairs"), py::kw_only(), py::arg("node_masks") = py::none(), py::arg("edge_masks") = py::none(), py::arg("flow_placement") = FlowPlacement::Proportional, py::arg("shortest_path") = false, py::arg("require_capacity") = true, py::arg("with_edge_flows") = false, py::arg("with_reachable") = false, py::arg("with_residuals") = false) - .def("sensitivity_analysis", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, std::int32_t dst, - FlowPlacement placement, bool shortest_path, bool require_capacity, - py::object node_mask, py::object edge_mask){ - if (src < 0 || src >= pg.num_nodes || dst < 0 || dst >= pg.num_nodes) throw py::value_error("src/dst out of range"); - MaxFlowOptions o; o.placement = placement; o.shortest_path = shortest_path; o.require_capacity = require_capacity; - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - o.node_mask = node_bs.view; o.edge_mask = edge_bs.view; - py::gil_scoped_release rel; auto res = algs.sensitivity_analysis(pg.handle, src, dst, o); py::gil_scoped_acquire acq; - // Return as list of tuples (EdgeId, FlowDelta) - py::list out; - for (auto const& pr : res) { - out.append(py::make_tuple(pr.first, pr.second)); - } - return out; - }, py::arg("graph"), py::arg("src"), py::arg("dst"), py::kw_only(), py::arg("flow_placement") = FlowPlacement::Proportional, py::arg("shortest_path") = false, py::arg("require_capacity") = true, py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()); - - py::class_(m, "PredDAG") - .def_property_readonly("parent_offsets", [](const PredDAG& d){ - return copy_to_numpy(std::span(d.parent_offsets.data(), d.parent_offsets.size())); - }) - .def_property_readonly("parents", [](const PredDAG& d){ - return copy_to_numpy(std::span(d.parents.data(), d.parents.size())); - }) - .def_property_readonly("via_edges", [](const PredDAG& d){ - return copy_to_numpy(std::span(d.via_edges.data(), d.via_edges.size())); - }) - .def_static("from_edges", [](const StrictMultiDiGraph& g, py::sequence edges){ - std::vector e; - e.reserve(py::len(edges)); - for (auto item : edges) e.push_back(py::cast(item)); - return make_path_dag(g, e); - }, py::arg("graph"), py::arg("edges"), - "Build a single-path PredDAG from a contiguous edge-id sequence.\n" - "Validates that each edge exists, consecutive edges connect, and the walk " - "is a simple path. The result works anywhere a PredDAG is accepted " - "(place_on_dag, FlowGraph.place, FlowPolicy.set_static_paths).") - .def("resolve_to_paths", [](const PredDAG& dag, std::int32_t src, std::int32_t dst, bool split_parallel_edges, py::object max_paths){ - std::optional mp; - if (!max_paths.is_none()) mp = py::cast(max_paths); - auto out = netgraph::core::resolve_to_paths(dag, src, dst, split_parallel_edges, mp); - py::list paths; - for (auto& path : out) { - py::list py_elems; - for (auto& pr : path) { - py::tuple edge_tuple(pr.second.size()); - for (std::size_t i=0;i(m, "MinCut") - .def_property_readonly("edges", [](const MinCut& mc){ - return copy_to_numpy(std::span(mc.edges.data(), mc.edges.size())); - }); - - py::class_(m, "FlowSummary") - .def_readonly("total_flow", &FlowSummary::total_flow) - .def_readonly("min_cut", &FlowSummary::min_cut) - .def_property_readonly("costs", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.costs.data(), s.costs.size())); - }) - .def_property_readonly("flows", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.flows.data(), s.flows.size())); - }) - .def_property_readonly("edge_flows", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.edge_flows.data(), s.edge_flows.size())); - }) - .def_property_readonly("residual_capacity", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.residual_capacity.data(), s.residual_capacity.size())); - }) - .def_property_readonly("reachable_nodes", [](const FlowSummary& s){ - // Store as bool array for Python; cast bytes to bool - py::array_t arr(s.reachable_nodes.size()); - auto* out = arr.mutable_data(); - for (std::size_t i=0;i(s.reachable_nodes[i]); - return arr; - }); - - // FlowState bindings - py::class_(m, "FlowState") - .def(py::init([](const StrictMultiDiGraph& g){ return FlowState(g); }), - py::arg("graph"), py::keep_alive<1, 2>()) - .def(py::init([](const StrictMultiDiGraph& g, py::array residual){ - if (!(py::isinstance>(residual))) throw py::type_error("residual must be a numpy float64 array"); - if (!(residual.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous"); - auto buf = residual.request(); - if (buf.ndim != 1 || static_cast(buf.shape[0]) != static_cast(g.num_edges())) throw py::type_error("residual length must equal num_edges"); - std::span rspan(static_cast(buf.ptr), static_cast(buf.shape[0])); - return FlowState(g, rspan); - }), py::arg("graph"), py::arg("residual"), py::keep_alive<1, 2>()) - .def("reset", [](FlowState& fs){ fs.reset(); }) - .def("reset", [](FlowState& fs, py::array residual){ - if (!(py::isinstance>(residual))) throw py::type_error("residual must be a numpy float64 array"); - if (!(residual.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous"); - auto buf = residual.request(); - if (buf.ndim != 1 || static_cast(buf.shape[0]) != static_cast(fs.capacity_view().size())) throw py::type_error("residual length must equal num_edges"); - std::span rspan(static_cast(buf.ptr), static_cast(buf.shape[0])); - fs.reset(rspan); - }) - .def("capacity_view", [](py::object self_obj){ - const FlowState& fs = py::cast(self_obj); - auto s = fs.capacity_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("residual_view", [](py::object self_obj){ - const FlowState& fs = py::cast(self_obj); - auto s = fs.residual_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("edge_flow_view", [](py::object self_obj){ - const FlowState& fs = py::cast(self_obj); - auto s = fs.edge_flow_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("place_on_dag", [](FlowState& fs, std::int32_t src, std::int32_t dst, const PredDAG& dag, double requested_flow, FlowPlacement placement){ - py::gil_scoped_release rel; auto placed = fs.place_on_dag(src, dst, dag, requested_flow, placement); py::gil_scoped_acquire acq; return placed; - }, py::arg("src"), py::arg("dst"), py::arg("dag"), py::arg("requested_flow") = std::numeric_limits::infinity(), py::arg("flow_placement") = FlowPlacement::Proportional) - .def("place_max_flow", [](py::object self_obj, std::int32_t src, std::int32_t dst, - FlowPlacement placement, bool shortest_path, bool require_capacity, - py::object node_mask, py::object edge_mask){ - FlowState& fs = py::cast(self_obj); - const StrictMultiDiGraph& g = fs.graph(); - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(g.num_nodes()), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(g.num_edges()), "edge_mask"); - py::gil_scoped_release rel; - auto total = fs.place_max_flow(src, dst, placement, shortest_path, require_capacity, - node_bs.view, edge_bs.view); - py::gil_scoped_acquire acq; - return total; - }, py::arg("src"), py::arg("dst"), - py::arg("flow_placement") = FlowPlacement::Proportional, - py::arg("shortest_path") = false, py::arg("require_capacity") = true, - py::kw_only(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()) - .def("compute_min_cut", [](py::object self_obj, std::int32_t src, py::object node_mask, py::object edge_mask){ - const FlowState& fs = py::cast(self_obj); - const StrictMultiDiGraph& g = fs.graph(); - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(g.num_nodes()), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(g.num_edges()), "edge_mask"); - py::gil_scoped_release rel; - auto mc = fs.compute_min_cut(src, node_bs.view, edge_bs.view); - py::gil_scoped_acquire acq; - return mc; - }, py::arg("src"), py::kw_only(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()); - - // FlowIndex and FlowGraph bindings - py::class_(m, "FlowIndex") - .def(py::init()) - .def_readonly("src", &FlowIndex::src) - .def_readonly("dst", &FlowIndex::dst) - .def_readonly("flowClass", &FlowIndex::flowClass) - .def_readonly("flowId", &FlowIndex::flowId); - - py::class_(m, "FlowGraph") - .def(py::init([](const StrictMultiDiGraph& g){ return FlowGraph(g); }), - py::arg("graph"), py::keep_alive<1, 2>()) - .def("capacity_view", [](py::object self_obj){ - const FlowGraph& fg = py::cast(self_obj); - auto s = fg.capacity_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("residual_view", [](py::object self_obj){ - const FlowGraph& fg = py::cast(self_obj); - auto s = fg.residual_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("edge_flow_view", [](py::object self_obj){ - const FlowGraph& fg = py::cast(self_obj); - auto s = fg.edge_flow_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def_property_readonly("graph", [](const FlowGraph& fg){ return &fg.graph(); }, py::return_value_policy::reference_internal) - .def("place", [](FlowGraph& fg, const FlowIndex& idx, std::int32_t src, std::int32_t dst, const PredDAG& dag, double amount, FlowPlacement placement){ py::gil_scoped_release rel; auto placed = fg.place(idx, src, dst, dag, amount, placement); py::gil_scoped_acquire acq; return placed; }, py::arg("index"), py::arg("src"), py::arg("dst"), py::arg("dag"), py::arg("amount"), py::arg("flow_placement") = FlowPlacement::Proportional) - .def("place_with_drops", [](FlowGraph& fg, const FlowIndex& idx, std::int32_t src, std::int32_t dst, const PredDAG& dag, double amount, FlowPlacement placement){ - std::vector> drops; - double placed; - { py::gil_scoped_release rel; placed = fg.place(idx, src, dst, dag, amount, placement, &drops); } - py::list out; for (auto const& pr : drops) out.append(py::make_tuple(pr.first, pr.second)); - return py::make_tuple(placed, out); - }, py::arg("index"), py::arg("src"), py::arg("dst"), py::arg("dag"), py::arg("amount"), py::arg("flow_placement") = FlowPlacement::EqualBalancedLossy, - "Like place(), but also returns the per-edge dropped volume as a list of (edge_id, dropped) pairs. Only EQUAL_BALANCED_LOSSY drops flow; other placements return an empty list.") - .def("remove", [](FlowGraph& fg, const FlowIndex& idx){ py::gil_scoped_release rel; fg.remove(idx); py::gil_scoped_acquire acq; }) - .def("remove_by_class", [](FlowGraph& fg, std::int32_t cls){ py::gil_scoped_release rel; fg.remove_by_class(cls); py::gil_scoped_acquire acq; }) - .def("reset", [](FlowGraph& fg){ py::gil_scoped_release rel; fg.reset(); py::gil_scoped_acquire acq; }) - .def("get_flow_edges", [](const FlowGraph& fg, const FlowIndex& idx){ auto v = fg.get_flow_edges(idx); py::list out; for (auto const& pr : v) { out.append(py::make_tuple(pr.first, pr.second)); } return out; }) - .def("get_flow_path", [](const FlowGraph& fg, const FlowIndex& idx){ auto v = fg.get_flow_path(idx); py::list out; for (auto e : v) out.append(e); return out; }); - - // NOTE: FlowRecord is deliberately not bound. No binding constructs or returns one - // (FlowPolicy.flows yields plain tuples), so the class was unreachable, and the - // Python name "Flow" collided with the C++ alias `using Flow = double`. If per-flow - // records become part of the public API, bind it under an unambiguous name such as - // "FlowRecord" and have FlowPolicy.flows return it instead of tuples. - - py::enum_(m, "PathAlg") - .value("SPF", PathAlg::SPF); - - py::class_(m, "FlowPolicyConfig") - .def(py::init<>()) - .def(py::init([](PathAlg path_alg, - FlowPlacement flow_placement, - EdgeSelection selection, - bool require_capacity, - bool multipath, - int min_flow_count, - py::object max_flow_count, - py::object max_path_cost, - py::object max_path_cost_factor, - bool shortest_path, - bool reoptimize_flows_on_each_placement, - int max_no_progress_iterations, - int max_total_iterations, - bool diminishing_returns_enabled, - int diminishing_returns_window, - double diminishing_returns_epsilon_frac){ - FlowPolicyConfig cfg; - cfg.path_alg = path_alg; - cfg.flow_placement = flow_placement; - cfg.selection = selection; - cfg.require_capacity = require_capacity; - cfg.multipath = multipath; - cfg.min_flow_count = min_flow_count; - if (!max_flow_count.is_none()) cfg.max_flow_count = py::cast(max_flow_count); - if (!max_path_cost.is_none()) cfg.max_path_cost = py::cast(max_path_cost); - if (!max_path_cost_factor.is_none()) cfg.max_path_cost_factor = py::cast(max_path_cost_factor); - cfg.shortest_path = shortest_path; - cfg.reoptimize_flows_on_each_placement = reoptimize_flows_on_each_placement; - cfg.max_no_progress_iterations = max_no_progress_iterations; - cfg.max_total_iterations = max_total_iterations; - cfg.diminishing_returns_enabled = diminishing_returns_enabled; - cfg.diminishing_returns_window = diminishing_returns_window; - cfg.diminishing_returns_epsilon_frac = diminishing_returns_epsilon_frac; - return cfg; - }), - py::kw_only(), - py::arg("path_alg") = PathAlg::SPF, - py::arg("flow_placement") = FlowPlacement::Proportional, - py::arg("selection") = EdgeSelection{}, - py::arg("require_capacity") = true, - py::arg("multipath") = true, - py::arg("min_flow_count") = 1, - py::arg("max_flow_count") = py::none(), - py::arg("max_path_cost") = py::none(), - py::arg("max_path_cost_factor") = py::none(), - py::arg("shortest_path") = false, - py::arg("reoptimize_flows_on_each_placement") = false, - py::arg("max_no_progress_iterations") = 100, - py::arg("max_total_iterations") = 10000, - py::arg("diminishing_returns_enabled") = true, - py::arg("diminishing_returns_window") = 8, - py::arg("diminishing_returns_epsilon_frac") = 1e-3) - .def_readwrite("path_alg", &FlowPolicyConfig::path_alg) - .def_readwrite("flow_placement", &FlowPolicyConfig::flow_placement) - .def_readwrite("selection", &FlowPolicyConfig::selection) - .def_readwrite("require_capacity", &FlowPolicyConfig::require_capacity) - .def_readwrite("multipath", &FlowPolicyConfig::multipath) - .def_readwrite("min_flow_count", &FlowPolicyConfig::min_flow_count) - .def_readwrite("max_flow_count", &FlowPolicyConfig::max_flow_count) - .def_readwrite("max_path_cost", &FlowPolicyConfig::max_path_cost) - .def_readwrite("max_path_cost_factor", &FlowPolicyConfig::max_path_cost_factor) - .def_readwrite("shortest_path", &FlowPolicyConfig::shortest_path) - .def_readwrite("reoptimize_flows_on_each_placement", &FlowPolicyConfig::reoptimize_flows_on_each_placement) - .def_readwrite("max_no_progress_iterations", &FlowPolicyConfig::max_no_progress_iterations) - .def_readwrite("max_total_iterations", &FlowPolicyConfig::max_total_iterations) - .def_readwrite("diminishing_returns_enabled", &FlowPolicyConfig::diminishing_returns_enabled) - .def_readwrite("diminishing_returns_window", &FlowPolicyConfig::diminishing_returns_window) - .def_readwrite("diminishing_returns_epsilon_frac", &FlowPolicyConfig::diminishing_returns_epsilon_frac); - - py::class_(m, "FlowPolicy") - .def("__init__", [](FlowPolicy& self, py::object algs_obj, const PyGraph& pg, FlowPolicyConfig cfg, - py::object node_mask, py::object edge_mask){ - if (!py::isinstance(algs_obj)) { - throw py::type_error("FlowPolicy: algorithms must be an Algorithms instance"); - } - std::shared_ptr algs = py::cast>(algs_obj); - - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - - cfg.node_mask = node_bs.view; - cfg.edge_mask = edge_bs.view; - - ExecutionContext ctx(algs, pg.handle); - new (&self) FlowPolicy(ctx, cfg); - }, - py::arg("algorithms"), py::arg("graph"), py::arg("config"), - py::kw_only(), - py::arg("node_mask") = py::none(), - py::arg("edge_mask") = py::none(), - py::keep_alive<1, 2>(), // self keeps algorithms alive - py::keep_alive<1, 3>()) // self keeps graph alive - .def("flow_count", &FlowPolicy::flow_count) - .def("placed_demand", &FlowPolicy::placed_demand) - .def("place_demand", [](FlowPolicy& p, FlowGraph& fg, std::int32_t src, std::int32_t dst, FlowClass flowClass, double volume, py::object target_per_flow, py::object min_flow){ std::optional tpf; if (!target_per_flow.is_none()) tpf = py::cast(target_per_flow); std::optional mfl; if (!min_flow.is_none()) mfl = py::cast(min_flow); py::gil_scoped_release rel; auto pr = p.place_demand(fg, src, dst, flowClass, volume, tpf, mfl); py::gil_scoped_acquire acq; return py::make_tuple(pr.first, pr.second); }, py::arg("flow_graph"), py::arg("src"), py::arg("dst"), py::arg("flowClass"), py::arg("volume"), py::arg("target_per_flow") = py::none(), py::arg("min_flow") = py::none()) - .def("rebalance_demand", [](FlowPolicy& p, FlowGraph& fg, std::int32_t src, std::int32_t dst, FlowClass flowClass, double target){ py::gil_scoped_release rel; auto pr = p.rebalance_demand(fg, src, dst, flowClass, target); py::gil_scoped_acquire acq; return py::make_tuple(pr.first, pr.second); }, - py::arg("flow_graph"), py::arg("src"), py::arg("dst"), py::arg("flowClass"), py::arg("target")) - .def("remove_demand", [](FlowPolicy& p, FlowGraph& fg){ py::gil_scoped_release rel; p.remove_demand(fg); py::gil_scoped_acquire acq; }) - .def("set_static_paths", [](FlowPolicy& p, std::int32_t src, std::int32_t dst, py::sequence paths){ - std::vector bundles; - bundles.reserve(py::len(paths)); - for (auto item : paths) bundles.push_back(py::cast(item)); - py::gil_scoped_release rel; - p.set_static_paths(src, dst, std::move(bundles)); - py::gil_scoped_acquire acq; - }, py::arg("src"), py::arg("dst"), py::arg("paths"), - "Pin this policy's demand to explicit path bundles (one flow per usable " - "bundle, bound in supply order). See FlowPolicy docs for validation, mask " - "(failure) semantics, and which config knobs become inert.") - .def_property_readonly("flows", [](const FlowPolicy& p){ py::dict out; for (auto const& kv : p.flows()) { const auto& idx = kv.first; const auto& f = kv.second; out[py::make_tuple(idx.src, idx.dst, idx.flowClass, idx.flowId)] = py::make_tuple(f.src, f.dst, f.cost, f.placed_flow); } return out; }); - - // Profiling functions (enabled via NGRAPH_CORE_PROFILE=1 environment variable) - m.def("profiling_enabled", &profiling_enabled, - "Return True if profiling is enabled (NGRAPH_CORE_PROFILE=1)."); - m.def("profiling_dump", []{ ProfilingStats::instance().dump(); }, - "Print profiling statistics to stderr."); - m.def("profiling_reset", []{ ProfilingStats::instance().reset(); }, - "Clear all profiling statistics."); + m.doc() = + "NetGraph-Core: the graph on integer units, the route structures, the integer ledger, the policy\n" + "engine FlowPolicy, the DagCache, the evaluators (capacity, batch_capacity, sensitivity) and the\n" + "searches (spf, ksp, resolve_to_paths).\n" + "Every public call either completes or raises and leaves Core state as it was, with two limits:\n" + "(1) pybind11 does not check every allocation when it constructs a bound object from Python, or\n" + "converts an attribute or a returned value into a new Python object outside the library's checked\n" + "result construction; running out of memory exactly there may crash the process, with Core state\n" + "unchanged. (2) A DagCache passed to the call: a building one keeps the complete entries and\n" + "statistics of the lookups that succeeded before the call failed; a frozen one counts one miss\n" + "when the call raised DagCacheMiss."; + + // In this order: a later binding's defaults and conversions need the earlier types. + bindings::register_axes(m); + bindings::register_units(m); + bindings::register_eligibility(m); + bindings::register_graph(m); + bindings::register_recorder(m); + bindings::register_pred_dag(m); + bindings::register_path(m); + bindings::register_flow_record(m); + bindings::register_flow_ledger(m); + bindings::register_routing(m); + bindings::register_policy_config(m); + bindings::register_placement_result(m); + bindings::register_dag_cache(m); + bindings::register_flow_policy(m); + bindings::register_evaluators(m); + bindings::register_paths(m); + bindings::register_search(m); +#if NETGRAPH_CORE_PRIVATE_HOOKS + bindings::register_private_hooks(m); +#endif } diff --git a/bindings/python/path.cpp b/bindings/python/path.cpp new file mode 100644 index 0000000..5039a25 --- /dev/null +++ b/bindings/python/path.cpp @@ -0,0 +1,78 @@ +// Path: an explicit walk of hops of one graph. +#include +#include +#include + +#include +#include +#include +#include + +#include "netgraph/core/graph.hpp" +#include "netgraph/core/pred_dag.hpp" +#include "integers.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_path(py::module_& m) { + py::class_>( + m, "Path", + "Immutable route of one graph, hop by hop: each hop lists one or more parallel edges (members)\n" + "from the same u to the same v, in the caller's order. cost = the sum over hops of the cost of\n" + "each hop's cheapest member.") + .def(py::init([](const std::shared_ptr& graph, const py::object& hops) { + if (!graph) throw py::value_error("graph must be a Graph"); + if (not_a_sequence(hops)) throw py::value_error("Path: hops must be a sequence of hops (sequences of EdgeIds)"); + auto seq = py::reinterpret_borrow(hops); + std::vector raw; + for (std::size_t h = 0; h < seq.size(); ++h) { + py::object hop = seq[h]; + raw.push_back(wide_integers(hop, ("hops[" + std::to_string(h) + "]").c_str())); + } + // Every member in range before anything else (no id is narrowed unchecked), then the + // C++ constructor's order. + const auto edge_count = static_cast(graph->num_edges()); + for (std::size_t h = 0; h < raw.size(); ++h) { + for (std::size_t i = 0; i < raw[h].size(); ++i) { + if (!raw[h].in(i, 0, edge_count - 1)) { + throw py::value_error("Path: hop " + std::to_string(h) + " member " + raw[h].text(i) + + " out of edge range [0, " + std::to_string(edge_count) + ")"); + } + } + } + std::vector> narrowed; + narrowed.reserve(raw.size()); + for (const auto& r : raw) narrowed.push_back(r.narrow()); + return std::make_shared(*graph, std::move(narrowed)); + }), + py::arg("graph"), py::arg("hops"), + "Validates, in order (ValueError on the first defect): every member id in range, no duplicate\n" + "member, a non-empty hop list, every hop non-empty, the members of a hop sharing (u, v),\n" + "contiguous hops, no repeated node.") + .def("graph_uid", &Path::graph_uid, "uid of the graph this path belongs to") + .def("hops", [](const Path& p) { return p.hops(); }, "the hops as lists of EdgeIds (the caller's member order)") + .def("src", &Path::src, "the first node of the walk") + .def("dst", &Path::dst, "the last node of the walk") + .def("cost", &Path::cost, "sum over hops of the minimum member cost") + .def( + "to_dag", [](const Path& p) { return std::make_shared(p.to_dag()); }, + "the equivalent PredDAG (every member of every hop)") + .def("__eq__", [](const Path& a, const py::object& b) { + return py::isinstance(b) && a == b.cast(); + }) + .def("__hash__", + [](const Path& a) { + py::list hops; + for (const auto& hop : a.hops()) hops.append(py::tuple(py::cast(hop))); + return py::hash(py::make_tuple(a.graph_uid(), py::tuple(hops))); + }) + .def("__repr__", [](const Path& a) { + return "Path(src=" + std::to_string(a.src()) + ", dst=" + std::to_string(a.dst()) + + ", hops=" + std::to_string(a.hops().size()) + ", cost=" + std::to_string(a.cost()) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/paths.cpp b/bindings/python/paths.cpp new file mode 100644 index 0000000..ba8b2c3 --- /dev/null +++ b/bindings/python/paths.cpp @@ -0,0 +1,183 @@ +// The path listings: ksp with KspOptions, and resolve_to_paths. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/enum_domain.hpp" +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/traversal.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/options.hpp" +#include "netgraph/core/pred_dag.hpp" +#include "netgraph/core/search.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// One NodeId of [0, num_nodes) (a single endpoint); the range message of detail/endpoints.hpp. +NodeId endpoint(const py::object& obj, std::int32_t num_nodes, const char* name) { + const py::object idx = python_int(obj, name); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= num_nodes) { + throw py::value_error(detail::node_range_error(name, std::string(py::str(idx)), num_nodes)); + } + return static_cast(v); +} + +// A count of at least 1 (KSP's k, the resolver's max_paths): the validation message below 1; a count +// above INT64_MAX means the same as INT64_MAX (no list reaches it). +std::int64_t count_of(const py::object& obj, const char* name) { + const py::object idx = python_int(obj, name); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow < 0 || (overflow == 0 && v < 1)) { + throw py::value_error(std::string(name) + " must be >= 1 (got " + std::string(py::str(idx)) + ")"); + } + return overflow != 0 ? std::numeric_limits::max() : static_cast(v); +} + +// The Python form of KspOptions: the arguments as given (k, max_path_cost and residual are +// validated at the call, in ksp's validation order, on their unnarrowed values). +struct PyKspOptions { + py::object k; + PathSelection selection; + py::object max_path_cost; + py::object residual; + std::shared_ptr elig; +}; + +} // namespace + +void register_paths(py::module_& m) { + py::class_( + m, "KspOptions", + "The options of ksp: k (>= 1) and selection are explicit (multi_edge = True lists each parallel\n" + "edge as a separate path, False keeps one per (u, v) pair as EdgeTieBreak describes; multipath is\n" + "ignored); max_path_cost (None, Absolute(c) with c >= 0, or Stretch(f) with f finite in [1, inf)\n" + "times the static shortest cost under the Eligibility, residual ignored; inclusive); residual\n" + "(int64 units per EdgeId, each in [0, capacity_units]: edges with less than 1 unit are skipped, and\n" + "the values feed the priority); elig (None: everything allowed). The fields are validated by ksp,\n" + "in the order its docstring gives.") + .def(py::init([](const py::object& k, const PathSelection& selection, const py::object& max_path_cost, + const py::object& residual, const std::shared_ptr& elig) { + return PyKspOptions {k, selection, max_path_cost, residual, elig}; + }), + py::arg("k"), py::arg("selection"), py::kw_only(), py::arg("max_path_cost") = py::none(), + py::arg("residual") = py::none(), py::arg("elig") = nullptr) + .def_readonly("k", &PyKspOptions::k, "the number of paths as given (validated by ksp: an integer >= 1)") + .def_readonly("selection", &PyKspOptions::selection, "the PathSelection (multipath is ignored)") + .def_readonly("max_path_cost", &PyKspOptions::max_path_cost, + "the inclusive cost ceiling as given: None, Absolute or Stretch (validated by ksp)") + .def_readonly("residual", &PyKspOptions::residual, + "the residual units per EdgeId as given (validated by ksp); None: static capacities, no\n" + "residual gating (edges below 1 unit of capacity are still skipped)") + .def_readonly("elig", &PyKspOptions::elig, "the static Eligibility; None: everything allowed"); + + m.def( + "ksp", + [](const std::shared_ptr& graph, const py::object& src, const py::object& dst, const py::object& opts) { + if (!graph) throw py::value_error("graph must be a Graph"); + const auto o = options_of(opts, "KspOptions"); + const auto n = graph->num_nodes(); + // ksp's validation order on unnarrowed values (Core repeats every check): the selection, src, + // dst, src == dst, k, the ceiling, the residual, then a foreign Eligibility. + detail::check_selection(o.selection); + const NodeId s = endpoint(src, n, "src"); + const NodeId t = endpoint(dst, n, "dst"); + if (s == t) throw py::value_error(detail::overlap_error(s, "src", "dst")); + KspOptions c(count_of(o.k, "k"), o.selection); + if (!o.max_path_cost.is_none()) { + if (py::isinstance(o.max_path_cost)) { + c.max_path_cost = o.max_path_cost.cast(); + } else if (py::isinstance(o.max_path_cost)) { + c.max_path_cost = o.max_path_cost.cast(); + } else { + throw py::type_error("max_path_cost must be Absolute, Stretch or None (got " + + std::string(py::str(py::type::of(o.max_path_cost).attr("__name__"))) + ")"); + } + detail::check_ceiling(c.max_path_cost); + } + std::optional> res; + if (!o.residual.is_none()) res = edge_units(o.residual, *graph, "residual"); + if (res.has_value()) c.residual = std::span(*res); + c.elig = o.elig.get(); + std::vector> paths; + { + py::gil_scoped_release release; + paths = ksp(*graph, s, t, c); + } + // Every object through a checked allocation: a failure raises. + py::list out(paths.size()); + for (std::size_t i = 0; i < paths.size(); ++i) { + py::tuple item(2); + py::int_ cost(paths[i].first); + py::object path = object_from(std::move(paths[i].second)); + PyTuple_SET_ITEM(item.ptr(), 0, cost.release().ptr()); + PyTuple_SET_ITEM(item.ptr(), 1, path.release().ptr()); + PyList_SET_ITEM(out.ptr(), static_cast(i), item.release().ptr()); + } + return out; + }, + py::arg("graph"), py::arg("src"), py::arg("dst"), py::arg("opts"), + "The k shortest simple src -> dst paths over the eligible edges of at least 1 unit, as the options\n" + "further limit them -> [(cost, Path)], each hop a single edge. Order: lowest cost; then, under\n" + "PREFER_HIGHER_PRIORITY only, highest bottleneck (the smallest priority value along the path); then\n" + "fewest hops; then the smallest EdgeId sequence. The result is the first k paths of that order\n" + "within the inclusive cost ceiling. Validated in this order before any work (ValueError): src and\n" + "dst in range, src == dst, k >= 1, the ceiling, the residual (length, then every entry), a foreign\n" + "Eligibility. An excluded endpoint, no static route under Stretch, or no path gives []."); + + m.def( + "resolve_to_paths", + [](const std::shared_ptr& graph, const std::shared_ptr& dag, const py::object& src, + const py::object& dst, bool split_parallel_edges, const py::object& max_paths) { + if (!graph) throw py::value_error("graph must be a Graph"); + if (!dag) throw py::value_error("dag must be a PredDAG"); + if (dag->graph_uid() != graph->uid()) throw py::value_error("foreign PredDAG: built for another graph"); + const auto n = graph->num_nodes(); + const NodeId s = endpoint(src, n, "src"); + const NodeId t = endpoint(dst, n, "dst"); + if (s == t) throw py::value_error(detail::overlap_error(s, "src", "dst")); + std::optional mp; + if (!max_paths.is_none()) mp = count_of(max_paths, "max_paths"); + std::vector paths; + { + py::gil_scoped_release release; + paths = resolve_to_paths(*graph, *dag, s, t, split_parallel_edges, mp); + } + py::list out(paths.size()); + for (std::size_t i = 0; i < paths.size(); ++i) { + PyList_SET_ITEM(out.ptr(), static_cast(i), object_from(std::move(paths[i])).release().ptr()); + } + return out; + }, + py::arg("graph"), py::arg("dag"), py::arg("src"), py::arg("dst"), py::arg("split_parallel_edges") = false, + py::arg("max_paths") = py::none(), + "The src -> dst walks of a DAG -> [Path], in increasing (cost, hops, edge-id sequence):\n" + "a hop costs its cheapest member and is keyed by its lowest member id. split_parallel_edges = False\n" + "keeps every parallel member of a (u, v) pair in one hop (members ascending); True gives one path\n" + "per member combination, sorted by that path's own (cost, hops, edge-id sequence). max_paths\n" + "(>= 1) truncates; None lists every walk.\n" + "ValueError: a DAG of another graph, src or dst out of range, src == dst, max_paths < 1. An\n" + "endpoint outside the DAG, or no walk, gives []."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/placement_result.cpp b/bindings/python/placement_result.cpp new file mode 100644 index 0000000..e579dfb --- /dev/null +++ b/bindings/python/placement_result.cpp @@ -0,0 +1,107 @@ +// Termination and PlacementResult: the outcome of one placement call. +#include +#include +#include + +#include +#include +#include + +#include "netgraph/core/flow_policy.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +py::object remaining_obj(const std::optional& r) { + return r.has_value() ? py::object(py::int_(*r)) : py::object(py::float_(std::numeric_limits::infinity())); +} + +} // namespace + +void register_placement_result(py::module_& m) { + py::enum_(m, "Termination", + "How a call ended, per source (local_outcomes) and for the whole call (termination).\n" + "A source's offer is served when it is 0, when (LOSSLESS) the delivery equals it, or when (LOSSY,\n" + "finite offer) all of it was forwarded, drops included. A source's outcome is the first that\n" + "applies:\n" + " COMPLETE: served.\n" + " LIMIT_FLOWS: creating or adding flows was stopped by MAX_FLOWS_PER_CALL, or adding flows by a\n" + " CapacityAware max_flow_count above flow_count.\n" + " LIMIT_DIMINISHING: the diminishing guard stopped adding flows.\n" + " NO_ROUTE: the source has no flows after the call (Global: the maximum flow is 0 and nothing\n" + " was ever placed).\n" + " EXHAUSTED: any other unserved case (every flow full and no new route helps, a full fixed-size\n" + " set of flows, Global at the maximum flow, a LOSSY infinite offer).\n" + "Global and Pinned act as one source. The call's termination is COMPLETE for a request of 0 or\n" + "when every source is served, NO_ROUTE for a positive request with an empty roster, else the\n" + "highest outcome of the unserved sources in the order LIMIT_FLOWS > LIMIT_DIMINISHING >\n" + "EXHAUSTED > NO_ROUTE.") + .value("COMPLETE", Termination::Complete) + .value("LIMIT_FLOWS", Termination::LimitFlows) + .value("LIMIT_DIMINISHING", Termination::LimitDiminishing) + .value("NO_ROUTE", Termination::NoRoute) + .value("EXHAUSTED", Termination::Exhausted); + + py::class_( + m, "PlacementResult", + "One placement call: placed (units), remaining (request - placed; float('inf') for an infinite\n" + "request), termination, local_outcomes [(source, Termination)] per roster source, routes_changed,\n" + "weights_changed, and this call's drops [(EdgeId, units)]. cost_distribution: Global only, the\n" + "marginal cost curve [(cost, units)] of the policy's flow so far, else None: in ascending cost,\n" + "delivering `units` more raised the minimum total flow cost by `cost` per unit (more delivery can\n" + "reroute earlier flow, so these are not the costs of the final paths); a zero call leaves it\n" + "unchanged. Optional outputs, None unless requested:\n" + "edge_flows and residual (int64[E]), bottleneck, and min_cut (Global with bottleneck; None when the\n" + "solve did not reach the maximum flow).") + .def_readonly("placed", &PlacementResult::placed, "the units placed by this call") + .def_property_readonly( + "remaining", [](const PlacementResult& r) { return remaining_obj(r.remaining); }, + "the request minus placed, in units (an amount, not a status); float('inf') for an infinite request") + .def_readonly("termination", &PlacementResult::termination, "the call's outcome (Termination)") + .def_readonly("local_outcomes", &PlacementResult::local_outcomes, + "[(source, Termination)] for every roster source, ascending node id") + .def_readonly("routes_changed", &PlacementResult::routes_changed, + "True when this call committed a route change: a flow created (also at zero delivery), rerouted\n" + "or added, or a change in the edges a Global source's share uses; a candidate equal to the\n" + "current route is not a change") + .def_readonly("weights_changed", &PlacementResult::weights_changed, + "True when this call committed new weights for a rerouted WEIGHTED flow") + .def_property_readonly( + "drops", [](const PlacementResult& r) { return pairs_list(r.drops); }, + "this call's transit drops [(EdgeId, units)], ascending EdgeId (empty unless Lossy; records\n" + "keep the cumulative drops)") + .def_readonly("cost_distribution", &PlacementResult::cost_distribution, + "Global: the marginal cost curve [(cost, units)] of the policy's cumulative flow (a zero\n" + "call reports it unchanged); None for the other routing modes") + .def_property_readonly( + "edge_flows", + [](const PlacementResult& r) -> py::object { + return r.edge_flows ? py::object(int64_array(*r.edge_flows)) : py::object(py::none()); + }, + "if requested: the policy's own allocation per edge after the call (int64[E], EdgeId order),\n" + "else None") + .def_property_readonly( + "residual", + [](const PlacementResult& r) -> py::object { + return r.residual ? py::object(int64_array(*r.residual)) : py::object(py::none()); + }, + "if requested: the ledger's residual after the call (int64[E], EdgeId order), else None") + .def_readonly("bottleneck", &PlacementResult::bottleneck, + "if requested: the ledger's saturation report after the call (FlowLedger.bottleneck of the\n" + "policy's sources and Eligibility), else None") + .def_readonly("min_cut", &PlacementResult::min_cut, + "Global with bottleneck requested: a minimum cut (edges whose removal separates the sources from\n" + "the destinations, with the smallest total capacity) of the policy's view (free capacity plus its\n" + "own allocation), when the solve reached the maximum flow; None otherwise (also on a zero call)") + .def("__repr__", [](const PlacementResult& r) { + return "PlacementResult(placed=" + std::to_string(r.placed) + ", termination=" + + std::string(py::str(py::cast(r.termination).attr("name"))) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/policy_config.cpp b/bindings/python/policy_config.cpp new file mode 100644 index 0000000..370b91d --- /dev/null +++ b/bindings/python/policy_config.cpp @@ -0,0 +1,68 @@ +// FlowPolicyConfig and OutputOptions: the configuration of a placement. +#include + +#include +#include +#include + +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/options.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +Routing routing_of(const py::object& obj) { + if (py::isinstance(obj)) return obj.cast(); + if (py::isinstance(obj)) return obj.cast(); + if (py::isinstance(obj)) return obj.cast(); + if (py::isinstance(obj)) return obj.cast(); + throw py::type_error("routing must be CostOnly, CapacityAware, Global or Pinned (got " + + std::string(py::str(py::type::of(obj).attr("__name__"))) + ")"); +} + +py::object routing_obj(const Routing& r) { + return std::visit([](const auto& x) { return py::cast(x); }, r); +} + +} // namespace + +void register_policy_config(py::module_& m) { + py::class_( + m, "FlowPolicyConfig", + "The four explicit axes (independent settings) of a policy: routing (CostOnly, CapacityAware,\n" + "Global or Pinned), placement (FlowPlacement), admission (Admission) and selection\n" + "(PathSelection). A placement or admission outside its enumeration raises ValueError.") + .def(py::init([](const py::object& routing, FlowPlacement placement, Admission admission, + const PathSelection& selection) { + // The routing's structure first (TypeError), then the constructor's enumeration checks. + auto r = routing_of(routing); + return FlowPolicyConfig {std::move(r), placement, admission, selection}; + }), + py::kw_only(), py::arg("routing"), py::arg("placement"), py::arg("admission"), py::arg("selection")) + .def_property_readonly( + "routing", [](const FlowPolicyConfig& c) { return routing_obj(c.routing); }, + "the routing mode: CostOnly, CapacityAware, Global or Pinned") + .def_readonly("placement", &FlowPolicyConfig::placement, "how every flow spreads over its route (FlowPlacement)") + .def_readonly("admission", &FlowPolicyConfig::admission, "LOSSLESS or LOSSY") + .def_readonly("selection", &FlowPolicyConfig::selection, "the PathSelection the routing's searches read") + .def("__repr__", [](const FlowPolicyConfig& c) { + return "FlowPolicyConfig(routing=" + std::string(py::repr(routing_obj(c.routing))) + ")"; + }); + + py::class_(m, "OutputOptions", + "Optional outputs of a placement (all false by default; absent outputs are None):\n" + "edge_flows (the policy's own allocation per edge after the call), residual and\n" + "bottleneck (the ledger's after the call); on Global, bottleneck adds min_cut.") + .def(py::init(), py::kw_only(), py::arg("edge_flows") = false, py::arg("residual") = false, + py::arg("bottleneck") = false) + .def_readonly("edge_flows", &OutputOptions::edge_flows, "request PlacementResult.edge_flows") + .def_readonly("residual", &OutputOptions::residual, "request PlacementResult.residual") + .def_readonly("bottleneck", &OutputOptions::bottleneck, + "request PlacementResult.bottleneck (and min_cut on Global)"); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/pred_dag.cpp b/bindings/python/pred_dag.cpp new file mode 100644 index 0000000..a388868 --- /dev/null +++ b/bindings/python/pred_dag.cpp @@ -0,0 +1,127 @@ +// PredDAG: the immutable predecessor DAG of one graph. +#include +#include +#include + +#include +#include +#include + +#include "netgraph/core/graph.hpp" +#include "netgraph/core/pred_dag.hpp" +#include "integers.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The exact order of wide integers: a positive overflow above every int64, a negative one +// below, and two overflows on the same side by their exact values. +bool wide_less(const WideInts& w, std::size_t i, std::size_t j) { + if (w.sign(i) != w.sign(j)) return w.sign(i) < w.sign(j); + if (w.sign(i) == 0) return w.value[i] < w.value[j]; + const int less = PyObject_RichCompareBool(w.exact(i).ptr(), w.exact(j).ptr(), Py_LT); + if (less < 0) throw py::error_already_set(); + return less == 1; +} + +} // namespace + +void register_pred_dag(py::module_& m) { + py::class_>( + m, "PredDAG", + "Immutable route structure of one graph, a predecessor DAG (directed, no cycle): for each node v,\n" + "the edges that reach v on the route and the node each comes from (its parent). The entries of v\n" + "are parents()[k], via_edges()[k] for k in [parent_offsets()[v], parent_offsets()[v + 1]), in\n" + "strictly ascending EdgeId, so two DAGs of the same graph with the same entries have equal arrays\n" + "and compare equal; DAGs of different graphs never compare equal.\n" + "parents() and via_edges() are read-only views that keep the DAG alive; parent_offsets() is a new\n" + "read-only array on each call (the DAG stores only the nodes that have entries). Built by\n" + "PredDAG.from_csr, Path.to_dag(), the searches and the policies (FlowRecord.bundle).") + .def_static( + "from_csr", + [](const std::shared_ptr& graph, const py::object& offsets, const py::object& parents, + const py::object& via) { + if (!graph) throw py::value_error("graph must be a Graph"); + // from_csr's validation order on unnarrowed values (the shape, then the ranges); the C++ + // constructor re-checks it and adds the structural checks (endpoints, duplicates, + // acyclic, ascending). + const auto n = static_cast(graph->num_nodes()); + const auto edge_count = static_cast(graph->num_edges()); + const std::size_t lo = sequence_length(offsets, "offsets"); + const std::size_t lp = sequence_length(parents, "parents"); + const std::size_t lv = sequence_length(via, "via"); + if (lo != n + 1) { + throw py::value_error("PredDAG.from_csr: offsets has length " + std::to_string(lo) + + ", expected N + 1 = " + std::to_string(n + 1)); + } + const auto o = wide_integers(offsets, "offsets"); + if (!o.in(0, 0, 0)) throw py::value_error("PredDAG.from_csr: offsets[0] = " + o.text(0) + ", expected 0"); + for (std::size_t v = 0; v < n; ++v) { + if (wide_less(o, v + 1, v)) { + throw py::value_error("PredDAG.from_csr: offsets are not monotone at " + std::to_string(v + 1) + " (" + + o.text(v) + " > " + o.text(v + 1) + ")"); + } + } + if (!o.in(n, static_cast(lp), static_cast(lp)) || lp != lv) { + throw py::value_error("PredDAG.from_csr: offsets[N] = " + o.text(n) + " but parents has " + + std::to_string(lp) + " and via has " + std::to_string(lv) + " entries"); + } + const auto p = wide_integers(parents, "parents"); + const auto w = wide_integers(via, "via"); + for (std::size_t k = 0; k < lp; ++k) { + if (!p.in(k, 0, static_cast(n) - 1)) { + throw py::value_error("PredDAG.from_csr: parents[" + std::to_string(k) + "] = " + p.text(k) + + " out of node range [0, " + std::to_string(n) + ")"); + } + if (!w.in(k, 0, edge_count - 1)) { + throw py::value_error("PredDAG.from_csr: via[" + std::to_string(k) + "] = " + w.text(k) + + " out of edge range [0, " + std::to_string(edge_count) + ")"); + } + } + const auto ov = o.narrow(); + const auto pv = p.narrow(); + const auto wv = w.narrow(); + return std::make_shared(PredDAG::from_csr(*graph, ov, pv, wv)); + }, + py::arg("graph"), py::arg("offsets"), py::arg("parents"), py::arg("via"), + "Validates, in order, before any traversal (ValueError on the first defect): offsets of\n" + "length N + 1, offsets[0] == 0, monotone, offsets[N] == len(parents) == len(via); then every\n" + "parent and via in range, every via joining (parent, child), no duplicate (child, via),\n" + "acyclic; within a node the entries in ascending via EdgeId.") + .def("graph_uid", &PredDAG::graph_uid, "uid of the graph this DAG belongs to") + .def( + "parent_offsets", + [](const PredDAG& d) { + const auto off = d.parent_offsets(); + py::array_t arr(static_cast(off.size())); + std::copy(off.begin(), off.end(), arr.mutable_data()); + py::detail::array_proxy(arr.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + return arr; + }, + "int32[N + 1]: the entries of node v are [parent_offsets[v], parent_offsets[v + 1]); a new read-only array per call") + .def( + "parents", [](const py::object& self) { return readonly_view(self.cast().parents(), self); }, + "int32 parent NodeId per entry (read-only view)") + .def( + "via_edges", + [](const py::object& self) { return readonly_view(self.cast().via_edges(), self); }, + "int32 via EdgeId per entry (read-only view)") + .def("__eq__", [](const PredDAG& a, const py::object& b) { + return py::isinstance(b) && a == b.cast(); + }) + .def("__hash__", [](const PredDAG& a) { + return py::hash(py::make_tuple(a.graph_uid(), py::tuple(py::cast(std::vector(a.via_edges().begin(), + a.via_edges().end()))))); + }) + .def("__repr__", [](const PredDAG& a) { + return "PredDAG(graph_uid=" + std::to_string(a.graph_uid()) + ", entries=" + + std::to_string(a.via_edges().size()) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/publication.hpp b/bindings/python/publication.hpp new file mode 100644 index 0000000..4809b01 --- /dev/null +++ b/bindings/python/publication.hpp @@ -0,0 +1,89 @@ +/* + The checked construction of result objects: the Python object of a bound class is allocated + through a checked tp_alloc, so an allocation failure raises instead of crashing, and a policy + call makes its result object before it changes anything. +*/ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include "netgraph/core/evaluators.hpp" +#include "netgraph/core/flow_policy.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// A new Python object of the bound class T owning a T made from `args`, made like pybind11's +// generic move-cast (type_caster_generic::cast) but with the object allocation checked: +// pybind11's make_new_instance uses the result of tp_alloc unchecked, so a failed Python +// allocation there is a crash rather than a MemoryError. Holder is the class's holder type +// (std::unique_ptr by default, std::shared_ptr for PredDAG and Path); another type is +// refused before anything is made. +// The steps, in order (new_object and object_from both use them): (1) allocate the object and +// check for null; (2) hold it in `obj` at once, so any later failure releases it; (3) allocate +// its layout; (4) make the value inside a local holder; (5) register the value and let the +// object take the holder; (6) mark the object as owner. Every failure raises and frees what was +// made exactly once. Until step 5 succeeds the object owns nothing: the local holder frees the +// value (std::shared_ptr also deletes it when its control block cannot be allocated), and the +// object is released without touching it. Taking the holder never allocates (pybind11 copies or +// moves it). +template +std::pair emplace_object(Args&&... args) { + static_assert(std::is_same_v> || std::is_same_v>); + const py::detail::type_info* tinfo = py::detail::get_type_info(typeid(T), /*throw_if_missing=*/true); + constexpr auto holder_kind = std::is_same_v> ? py::detail::holder_enum_t::std_shared_ptr + : py::detail::holder_enum_t::std_unique_ptr; + if (tinfo->holder_enum_v != holder_kind) { + py::pybind11_fail("emplace_object: Holder is not the holder of the bound class"); + } + PyTypeObject* type = tinfo->type; + PyObject* raw = type->tp_alloc(type, 0); + if (raw == nullptr) throw py::error_already_set(); + auto obj = py::reinterpret_steal(raw); + auto* inst = reinterpret_cast(raw); + inst->allocate_layout(); + inst->owned = false; + Holder holder(new T(std::forward(args)...)); + T* value = holder.get(); + py::detail::values_and_holders(inst).begin()->value_ptr() = value; + tinfo->init_instance(inst, &holder); + inst->owned = true; + return {std::move(obj), value}; +} + +// A new Python object of the bound class T (default holder) owning a default-constructed T +// (emplace_object). The policy calls make their result object with this before they change +// anything. +template +std::pair new_object() { + return emplace_object>(); +} + +// The value moved into its result object after the commit: nothing may allocate there. +static_assert(std::is_nothrow_move_assignable_v); +static_assert(std::is_nothrow_move_assignable_v); +static_assert(std::is_nothrow_move_assignable_v); + +template +py::object object_of(T value) { + auto [obj, slot] = new_object(); + *slot = std::move(value); + return std::move(obj); +} + +// A new Python object of the bound class T (held by std::shared_ptr) owning `value`, through the +// same checked allocation as new_object: for the values without a default constructor (PredDAG, +// Path) that a search returns. A failure raises and releases everything made so far. +template +py::object object_from(T value) { + return emplace_object>(std::move(value)).first; +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/recorder.cpp b/bindings/python/recorder.cpp new file mode 100644 index 0000000..0faea8a --- /dev/null +++ b/bindings/python/recorder.cpp @@ -0,0 +1,109 @@ +// The private entry _place_global_units, registered in both builds: the flow core's +// minimum-cost flow from zero on a given view, with its full payload for the flow checks. +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/flow_core.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +py::list int_list(const std::vector& v) { + py::list out(v.size()); + for (std::size_t i = 0; i < v.size(); ++i) out[i] = py::int_(v[i]); + return out; +} + +} // namespace + +void register_recorder(py::module_& m) { + m.def( + "_place_global_units", + [](const std::shared_ptr& graph, const py::object& view, const py::object& sources, + const py::object& destinations, const std::shared_ptr& elig, + const py::object& cap) -> py::dict { + if (!graph) throw py::value_error("graph must be a Graph"); + // The core's order (view length, view entries, sources, destinations, overlap, cap, + // Eligibility), every value checked before narrowing; the core re-checks the same order. + const auto v = edge_amounts(view, *graph, "view", "view (capacity units)"); + const auto s = checked_ids(wide_integers(sources, "sources node ids"), graph->num_nodes(), "sources"); + const auto d = checked_ids(wide_integers(destinations, "destinations node ids"), graph->num_nodes(), "destinations"); + detail::check_disjoint(detail::node_set(s, graph->num_nodes(), "sources"), + detail::node_set(d, graph->num_nodes(), "destinations"), "sources", "destinations"); + std::optional c; + if (!cap.is_none()) { + if (is_bool(cap) || !PyIndex_Check(cap.ptr())) { + throw py::value_error("cap must be an integer number of units in [0, 2^63 - 2] or None"); + } + py::object idx = python_int(cap, "cap"); + int overflow = 0; + const long long value = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || value < 0 || value > detail::kMaxFlowCap) { + throw py::value_error("cap " + std::string(py::str(idx)) + " is outside [0, 2^63 - 2]"); + } + c = static_cast(value); + } + detail::FlowCoreResult r; + { + py::gil_scoped_release release; + const auto view_units = v.narrow(); + r = detail::place_global_units(*graph, view_units, s, d, elig.get(), c); + } + py::dict out; + out["placed"] = py::int_(r.placed); + py::list curve; + for (const auto& [cost, units] : r.cost_distribution) { + py::list entry; + entry.append(py::int_(cost)); + entry.append(py::int_(units)); + curve.append(entry); + } + out["cost_distribution"] = curve; + out["edge_flows"] = int_list(r.edge_flows); + if (r.min_cut.has_value()) { + py::list cut; + for (EdgeId e : *r.min_cut) cut.append(py::int_(e)); + out["min_cut"] = cut; + } else { + out["min_cut"] = py::none(); + } + out["potentials"] = int_list(r.potentials); + py::dict counters; + counters["dijkstra_runs"] = py::int_(r.counters.dijkstra_runs); + counters["dijkstra_pops"] = py::int_(r.counters.dijkstra_pops); + counters["settled"] = py::int_(r.counters.settled); + counters["arc_scans"] = py::int_(r.counters.arc_scans); + counters["dinic_phases"] = py::int_(r.counters.dinic_phases); + counters["augmentations"] = py::int_(r.counters.augmentations); + out["counters"] = counters; + return out; + }, + py::arg("graph"), py::arg("view"), py::arg("sources"), py::arg("destinations"), py::arg("elig"), + py::arg("cap"), + "Private flow-core entry: the largest flow up to cap (None: no cap) from the source set to the\n" + "destination set over the eligible edges, sent at the lowest total cost, on the capacities `view`\n" + "(int64 units, EdgeId order), starting from zero flow. Returns {placed, cost_distribution:\n" + "[[cost, units]], edge_flows, min_cut (None while the destinations are still reachable, because\n" + "the cap stopped the solve, and on cap 0), potentials (per-node prices from the shortest-path runs\n" + "that prove the flow is cheapest; -1 = never reached), counters}. Invalid inputs raise ValueError\n" + "before any work. A solve (cap not 0) then raises ValueError, before the solver sizes anything, on\n" + "a network beyond the flow core's int32 numbering: more than INT32_MAX - 2 nodes, or more than\n" + "2^30 - 1 eligible edges, eligible sources and eligible destinations together."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/registration.hpp b/bindings/python/registration.hpp new file mode 100644 index 0000000..82020f2 --- /dev/null +++ b/bindings/python/registration.hpp @@ -0,0 +1,40 @@ +/* + The registration functions of the module _netgraph_core, one file each under bindings/python/. + The module entry (module.cpp) calls them in this order, which is the order of the module's + attributes: a default argument or a conversion of a later binding needs the types registered + before it. The private recorder entry (_place_global_units) is registered + in both builds; only register_private_hooks and the private members of FlowLedger and + FlowPolicy depend on NETGRAPH_CORE_PRIVATE_HOOKS. +*/ +#pragma once + +#include +#include +#include + +#include "netgraph/core/detail/hooks_flag.hpp" + +namespace netgraph::core::bindings { + +void register_axes(pybind11::module_& m); // the axes' enumerations, PathSelection +void register_units(pybind11::module_& m); // to_units, as_float +void register_eligibility(pybind11::module_& m); // Eligibility +void register_graph(pybind11::module_& m); // Graph +void register_recorder(pybind11::module_& m); // _place_global_units (both builds) +void register_pred_dag(pybind11::module_& m); // PredDAG +void register_path(pybind11::module_& m); // Path +void register_flow_record(pybind11::module_& m); // FlowRecord +void register_flow_ledger(pybind11::module_& m); // FlowLedger +void register_routing(pybind11::module_& m); // Absolute, Stretch, DiminishingReturns, the four routings +void register_policy_config(pybind11::module_& m); // FlowPolicyConfig, OutputOptions +void register_placement_result(pybind11::module_& m); // Termination, PlacementResult +void register_dag_cache(pybind11::module_& m); // DagCacheStats, DagCacheMiss, DagCache +void register_flow_policy(pybind11::module_& m); // FlowPolicy +void register_evaluators(pybind11::module_& m); // capacity, batch_capacity, sensitivity and their types +void register_paths(pybind11::module_& m); // KspOptions, ksp, resolve_to_paths +void register_search(pybind11::module_& m); // UNREACHABLE, SpfOptions, SpfResult, spf +#if NETGRAPH_CORE_PRIVATE_HOOKS +void register_private_hooks(pybind11::module_& m); // the test controls and diagnostics +#endif + +} // namespace netgraph::core::bindings diff --git a/bindings/python/routing.cpp b/bindings/python/routing.cpp new file mode 100644 index 0000000..807c558 --- /dev/null +++ b/bindings/python/routing.cpp @@ -0,0 +1,231 @@ +// The routing modes of a policy (CostOnly, CapacityAware, Global, Pinned) and their parameters: +// the cost ceilings Absolute and Stretch, the DiminishingReturns guard. +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/options.hpp" +#include "netgraph/core/pred_dag.hpp" +#include "integers.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// An optional int field of [lo, hi] (None allowed when `optional`); bools and floats rejected. +std::optional int_field(const py::object& obj, std::int64_t lo, std::int64_t hi, const std::string& what, + bool optional) { + if (obj.is_none()) { + if (optional) return std::nullopt; + throw py::value_error(what + " must be an integer (got None)"); + } + return integer_value(obj, lo, hi, what); +} + +// A float field: a Python float, an integer or a NumPy floating scalar; bools and other types +// rejected. The conversion order is Python's float(): a float as it is, then the object's own +// __float__, then its __index__. A Python integer beyond the finite doubles (the int itself, or the +// one its __index__ returned) is outside the field's domain (ValueError); every exception raised by +// the object's own __float__ or __index__, OverflowError included, propagates as raised. +double float_field(const py::object& obj, const std::string& what) { + if (is_bool(obj) || !(py::isinstance(obj) || PyIndex_Check(obj.ptr()) || + py::isinstance(obj, py::module_::import("numpy").attr("floating")))) { + throw py::value_error(what + " must be a number (got " + std::string(py::str(py::type::of(obj).attr("__name__"))) + + ")"); + } + if (PyFloat_Check(obj.ptr())) return PyFloat_AS_DOUBLE(obj.ptr()); + const PyNumberMethods* nb = Py_TYPE(obj.ptr())->tp_as_number; + const bool own_float = nb != nullptr && nb->nb_float != nullptr && nb->nb_float != PyLong_Type.tp_as_number->nb_float; + if (own_float) { // the object's own __float__ (a NumPy scalar, a class defining it) + const double x = PyFloat_AsDouble(obj.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + return x; + } + // A Python int (int's own __float__ only narrows it), or an object converted by its __index__. + py::object idx = py::reinterpret_borrow(obj); + if (!PyLong_Check(obj.ptr())) { + idx = py::reinterpret_steal(PyNumber_Index(obj.ptr())); + if (!idx) throw py::error_already_set(); + } + const double x = PyLong_AsDouble(idx.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) throw py::error_already_set(); + PyErr_Clear(); + throw py::value_error(what + " must be a number in the float range (got an integer beyond the largest finite float)"); + } + return x; +} + +} // namespace + +void register_routing(py::module_& m) { + py::class_(m, "Absolute", "Cost ceiling: an absolute, inclusive path-cost budget (cost >= 0).") + .def(py::init([](const py::object& cost) { + return Absolute {*int_field(cost, std::numeric_limits::min(), std::numeric_limits::max(), + "Absolute cost", false)}; + }), + py::arg("cost"), "Absolute(cost): a 64-bit integer budget in cost units (checked >= 0 where it is used)") + .def_readonly("cost", &Absolute::cost, "the budget: a path qualifies when its cost is at most this") + .def("__eq__", [](const Absolute& a, const py::object& b) { return py::isinstance(b) && a == b.cast(); }) + .def("__hash__", [](const Absolute& a) { return py::hash(py::make_tuple("Absolute", a.cost)); }) + .def("__repr__", [](const Absolute& a) { return "Absolute(" + std::to_string(a.cost) + ")"; }); + + py::class_(m, "Stretch", + "Cost ceiling: a route qualifies when its cost is at most factor times the static shortest cost\n" + "under the Eligibility; factor is in [1, inf), and the float is compared exactly, without rounding.") + .def(py::init([](const py::object& factor) { return Stretch {float_field(factor, "Stretch factor")}; }), + py::arg("factor"), + "Stretch(factor): a number, checked finite and >= 1 where it is used; a path qualifies when its\n" + "cost is at most factor times the static shortest cost. Here, an integer beyond the largest finite\n" + "float raises ValueError.") + .def_readonly("factor", &Stretch::factor, "the factor as a float") + .def("__eq__", [](const Stretch& a, const py::object& b) { return py::isinstance(b) && a == b.cast(); }) + .def("__hash__", [](const Stretch& a) { return py::hash(py::make_tuple("Stretch", a.factor)); }) + .def("__repr__", [](const Stretch& a) { return "Stretch(" + std::string(py::str(py::float_(a.factor))) + ")"; }); + + py::class_(m, "DiminishingReturns", + "The diminishing-returns guard of CapacityAware growth: stop once at least `window`\n" + "growth steps were accepted in the call and the last `window` gains sum below\n" + "`fraction` times the source's (finite) offer. window >= 1, fraction in (0, 1).") + .def(py::init([](const py::object& window, const py::object& fraction) { + DiminishingReturns d; + d.window = static_cast(*int_field(window, std::numeric_limits::min(), + std::numeric_limits::max(), + "diminishing window", false)); + d.fraction = float_field(fraction, "diminishing fraction"); + return d; + }), + py::arg("window"), py::arg("fraction"), + "DiminishingReturns(window, fraction): a 32-bit integer and a number, checked (window >= 1,\n" + "fraction in (0, 1)) when a policy is built with them. Here, a window outside 32 bits or a fraction\n" + "that is an integer beyond the largest finite float raises ValueError.") + .def_readonly("window", &DiminishingReturns::window, "the number of recent accepted growth steps compared") + .def_readonly("fraction", &DiminishingReturns::fraction, + "the share of the source's offer those steps must gain together to continue") + .def("__repr__", [](const DiminishingReturns& d) { + return "DiminishingReturns(window=" + std::to_string(d.window) + ", fraction=" + std::string(py::str(py::float_(d.fraction))) + ")"; + }); + + py::class_(m, "CostOnly", + "Routing like IGP forwarding (OSPF, IS-IS routers): one flow per roster source on the static\n" + "shortest-path DAG (read from the DagCache; the selection is applied when the flow is created),\n" + "never refreshed, grown or reoptimised.") + .def(py::init<>(), "CostOnly routing (no parameters)") + .def("__repr__", [](const CostOnly&) { return std::string("CostOnly()"); }); + + py::class_( + m, "CapacityAware", + "Routing like distributed CSPF tunnels (each tunnel head picks its own shortest path among edges\n" + "with enough free capacity). Each source creates flow_count flows, one search at a time, on edges\n" + "with residual >= max(1, min_flow_bandwidth) units and within max_path_cost; while its offer is\n" + "unmet it adds flows up to max_flow_count (None: no ceiling). With reoptimize, a flow moves to a\n" + "new route only when it keeps its whole amount there. `diminishing` (None: off) stops adding flows\n" + "once the recent ones gain too little.") + .def(py::init([](const py::object& flow_count, const py::object& max_flow_count, const py::object& max_path_cost, + const py::object& min_flow_bandwidth, bool reoptimize, const py::object& diminishing) { + CapacityAware ca; + constexpr auto lo32 = std::numeric_limits::min(); + constexpr auto hi32 = std::numeric_limits::max(); + ca.flow_count = static_cast(*int_field(flow_count, lo32, hi32, "flow_count", false)); + if (auto mx = int_field(max_flow_count, lo32, hi32, "max_flow_count", true)) { + ca.max_flow_count = static_cast(*mx); + } + if (!max_path_cost.is_none()) { + if (py::isinstance(max_path_cost)) { + ca.max_path_cost = max_path_cost.cast(); + } else if (py::isinstance(max_path_cost)) { + ca.max_path_cost = max_path_cost.cast(); + } else { + throw py::type_error("max_path_cost must be Absolute, Stretch or None"); + } + } + ca.min_flow_bandwidth = *int_field(min_flow_bandwidth, std::numeric_limits::min(), + std::numeric_limits::max(), "min_flow_bandwidth (units)", false); + ca.reoptimize = reoptimize; + if (!diminishing.is_none()) { + if (!py::isinstance(diminishing)) throw py::type_error("diminishing must be DiminishingReturns or None"); + ca.diminishing = diminishing.cast(); + } + return ca; + }), + py::kw_only(), py::arg("flow_count") = 1, py::arg("max_flow_count") = py::none(), + py::arg("max_path_cost") = py::none(), py::arg("min_flow_bandwidth") = 0, py::arg("reoptimize") = false, + py::arg("diminishing") = py::none(), + "The fields are checked when a policy is built with them (flow_count >= 1, max_flow_count >=\n" + "flow_count, min_flow_bandwidth in [0, 2^62) units, the ceiling, the guard). Here, an integer field\n" + "of the wrong type or outside its integer type (32-bit flow_count and max_flow_count, 64-bit\n" + "min_flow_bandwidth) raises ValueError; a ceiling or guard of the wrong class, a reoptimize that is\n" + "not a bool, a number or None (a number or None counts by its truth value), and an unknown keyword\n" + "raise TypeError.") + .def_readonly("flow_count", &CapacityAware::flow_count, + "the flows a source creates on its first positive request") + .def_readonly("max_flow_count", &CapacityAware::max_flow_count, + "the most flows growth may reach per source; None: no ceiling") + .def_property_readonly( + "max_path_cost", + [](const CapacityAware& ca) -> py::object { + if (!ca.max_path_cost.has_value()) return py::none(); + return std::visit([](const auto& x) { return py::cast(x); }, *ca.max_path_cost); + }, + "the inclusive route cost ceiling (Absolute or Stretch); None: no ceiling") + .def_readonly("min_flow_bandwidth", &CapacityAware::min_flow_bandwidth, + "units in [0, 2^62): every search keeps only edges with at least max(1, this) residual units") + .def_readonly("reoptimize", &CapacityAware::reoptimize, + "True: a retained flow moves to a newly found route when it keeps its whole amount there") + .def_readonly("diminishing", &CapacityAware::diminishing, "the growth guard; None: off") + .def("__repr__", [](const CapacityAware& ca) { + return "CapacityAware(flow_count=" + std::to_string(ca.flow_count) + ", max_flow_count=" + + (ca.max_flow_count ? std::to_string(*ca.max_flow_count) : std::string("None")) + ")"; + }); + + py::class_(m, "Global", + "Routing: an ideal central controller that sees the whole network. Every positive call re-solves\n" + "the minimum-cost flow of what the policy already delivers plus this call's request\n" + "(placed_demand() + volume; what an earlier call could not place is not requested again) over\n" + "the eligible graph, on the free capacity plus its own allocation (it may move its own traffic,\n" + "never another's). Placement MAX_FLOW only. A positive call on a network beyond the flow core's\n" + "int32 numbering raises ValueError (see FlowPolicy.place).") + .def(py::init<>(), "Global routing (no parameters)") + .def("__repr__", [](const Global&) { return std::string("Global()"); }); + + py::class_(m, "Pinned", + "Routing on routes the caller gives: one flow per usable bundle (a PredDAG with at least one\n" + "src -> dst walk over eligible edges of at least 1 unit of capacity), in the given order; one\n" + "source and one destination. A Path converts with to_dag(). The list is copied at construction.") + .def(py::init([](const py::object& bundles) { + if (not_a_sequence(bundles)) throw py::type_error("Pinned bundles must be a sequence of PredDAG"); + Pinned p; + for (const auto& b : py::reinterpret_borrow(bundles)) { + if (!py::isinstance(b)) { + throw py::type_error("Pinned bundles must be PredDAG values (a Path converts with to_dag())"); + } + p.bundles.push_back(b.cast>()); + } + return p; + }), + py::arg("bundles"), + "Pinned(bundles): a sequence of PredDAG of one graph, copied; the policy checks them when it is\n" + "built (non-empty, no duplicates, each with a src -> dst walk)") + .def_property_readonly( + "bundles", + [](const Pinned& p) { + py::list out; + for (const auto& b : p.bundles) out.append(std::const_pointer_cast(b)); + return out; + }, + "the bundles in supply order (a new list of the shared, immutable PredDAG values)") + .def("__repr__", [](const Pinned& p) { return "Pinned(bundles=" + std::to_string(p.bundles.size()) + ")"; }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/search.cpp b/bindings/python/search.cpp new file mode 100644 index 0000000..d4039d2 --- /dev/null +++ b/bindings/python/search.cpp @@ -0,0 +1,182 @@ +// spf with SpfOptions and SpfResult, the public search, and UNREACHABLE. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/enum_domain.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/options.hpp" +#include "netgraph/core/pred_dag.hpp" +#include "netgraph/core/search.hpp" +#include "netgraph/core/types.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The Python form of SpfOptions: the arguments as given (targets, residual and min_residual are +// validated at the call, in spf's validation order, on their unnarrowed values); the Eligibility is +// held by reference count. +struct PySpfOptions { + Direction direction; + PathSelection selection; + py::object targets; + py::object residual; + py::object min_residual; + std::shared_ptr elig; +}; + +// The Python form of SpfResult: every field already a Python object (made by the call through +// the checked allocations), so reading one allocates nothing. +struct PySpfResult { + py::object costs; // a read-only int64 array (full searches) or None + py::object dag; // PredDAG + py::object target_cost; // an int, or None (no target reached, or a full search) +}; + +} // namespace + +void register_search(py::module_& m) { + m.attr("UNREACHABLE") = py::int_(kUnreachable); + + py::class_( + m, "SpfOptions", + "The options of spf: direction and selection are explicit; targets (one NodeId or a sequence;\n" + "None: a full search) stops the search at the nearest targets (anycast: any nearest target will\n" + "do); residual (int64 units per EdgeId, each in [0, capacity_units]) makes the search skip edges\n" + "with fewer than min_residual residual units (an integer >= 1; without a residual it has no\n" + "effect) and supplies the priority values; elig (None: everything allowed). A direction outside its\n" + "enumeration raises ValueError here; the other fields are validated by spf, in the order its\n" + "docstring gives.") + .def(py::init([](Direction direction, const PathSelection& selection, const py::object& targets, + const py::object& residual, const py::object& min_residual, + const std::shared_ptr& elig) { + detail::check_member(direction, "direction"); + return PySpfOptions {direction, selection, targets, residual, min_residual, elig}; + }), + py::arg("direction"), py::arg("selection"), py::kw_only(), py::arg("targets") = py::none(), + py::arg("residual") = py::none(), py::arg("min_residual") = 1, py::arg("elig") = nullptr) + .def_readonly("direction", &PySpfOptions::direction, "FORWARD or REVERSE") + .def_readonly("selection", &PySpfOptions::selection, "the PathSelection the search reads") + .def_readonly("targets", &PySpfOptions::targets, "the targets as given (validated by spf); None: a full search") + .def_readonly("residual", &PySpfOptions::residual, + "the residual units per EdgeId as given (validated by spf); None: static capacities, no\n" + "residual gating (edges below 1 unit of capacity are still skipped)") + .def_readonly("min_residual", &PySpfOptions::min_residual, + "the gating threshold in units as given (validated by spf: an integer >= 1)") + .def_readonly("elig", &PySpfOptions::elig, "the static Eligibility; None: everything allowed"); + + py::class_(m, "SpfResult", + "The result of spf: costs (full searches: a read-only int64 array, UNREACHABLE for the nodes not\n" + "reached; None with targets), dag (PredDAG of the tight edges, the edges on some shortest walk,\n" + "one per (u, v) pair with multi_edge = False, each in its own direction; with targets only the\n" + "walks between the roots and a nearest target, for REVERSE from the target to a root; for\n" + "multipath = False the canonical single path) and\n" + "target_cost (with targets: the nearest targets' cost, None when no target is reached; None for a\n" + "full search).") + .def_property_readonly( + "costs", [](const PySpfResult& r) { return r.costs ? r.costs : py::object(py::none()); }, + "full search: each node's distance from the roots for FORWARD, or to the roots for REVERSE\n" + "(read-only int64[N], UNREACHABLE for nodes not reached); None with targets") + .def_property_readonly( + "dag", [](const PySpfResult& r) { return r.dag; }, + "the PredDAG of the result, in the edges' original orientation") + .def_property_readonly( + "target_cost", [](const PySpfResult& r) { return r.target_cost ? r.target_cost : py::object(py::none()); }, + "with targets: the cost of the nearest targets, None when no target is reached; None for a\n" + "full search") + .def("__repr__", [](const PySpfResult& r) { + std::string tc = r.target_cost && !r.target_cost.is_none() ? std::string(py::str(r.target_cost)) : "None"; + return "SpfResult(costs=" + std::string(r.costs ? "array" : "None") + + ", dag_entries=" + std::to_string(r.dag.cast().via_edges().size()) + + ", target_cost=" + tc + ")"; + }); + + m.def( + "spf", + [](const std::shared_ptr& graph, const py::object& roots, const py::object& opts) { + if (!graph) throw py::value_error("graph must be a Graph"); + const auto o = options_of(opts, "SpfOptions"); + const auto n = graph->num_nodes(); + // spf's validation order on unnarrowed values (Core repeats every check): the enumerations, + // the roots, the targets, their overlap, a single path without targets, the residual, then + // min_residual and a foreign Eligibility. + detail::check_member(o.direction, "direction"); + detail::check_selection(o.selection); + const auto r = node_ids(roots, n, "roots"); + std::optional> t; + if (!o.targets.is_none()) { + t = node_ids(o.targets, n, "targets"); + detail::check_disjoint(detail::node_set(r, n, "roots"), detail::node_set(*t, n, "targets"), "roots", + "targets"); + } else if (!o.selection.multipath) { + throw py::value_error("multipath = false requires targets (the single path joins a root to a target)"); + } + std::optional> res; + if (!o.residual.is_none()) res = edge_units(o.residual, *graph, "residual"); + const auto minr = python_int(o.min_residual, "min_residual"); + int overflow = 0; + const long long mv = PyLong_AsLongLongAndOverflow(minr.ptr(), &overflow); + if (overflow < 0 || (overflow == 0 && mv < 1)) { + throw py::value_error("min_residual must be at least 1 unit (got " + std::string(py::str(minr)) + ")"); + } + SpfOptions c(o.direction, o.selection); + c.targets = std::move(t); + if (res.has_value()) c.residual = std::span(*res); + // Above INT64_MAX it gates every residual edge exactly as INT64_MAX does (residuals < 2^62). + c.min_residual = overflow != 0 ? std::numeric_limits::max() : static_cast(mv); + c.elig = o.elig.get(); + std::optional result; + { + py::gil_scoped_release release; + result.emplace(spf(*graph, r, c)); + } + // The result's Python objects, each through a checked allocation: a failure raises. + auto [obj, slot] = new_object(); + if (result->costs.has_value()) { + py::array_t costs(static_cast(result->costs->size())); + std::copy(result->costs->begin(), result->costs->end(), costs.mutable_data()); + py::detail::array_proxy(costs.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + slot->costs = std::move(costs); + } + slot->dag = object_from(std::move(result->dag)); + if (result->target_cost.has_value() && *result->target_cost != kUnreachable) { + slot->target_cost = py::int_(*result->target_cost); + } + return std::move(obj); + }, + py::arg("graph"), py::arg("roots"), py::arg("opts"), + "Shortest paths from the roots for FORWARD, or to the roots for REVERSE (one NodeId or a sequence;\n" + "every root at distance 0), over the eligible edges of at least 1 unit of capacity (and of residual\n" + ">= min_residual with a residual) -> SpfResult. Without targets every reachable node gets its\n" + "distance: costs and the full DAG of tight edges (the edges on some shortest walk; one per (u, v)\n" + "pair with multi_edge = False). With targets the search stops at the nearest targets' cost:\n" + "target_cost and the DAG of the walks between the roots and a nearest target (for REVERSE, from\n" + "the target to a root, in the edges' own direction);\n" + "with multipath = False, instead the canonical single path, the walk minimising (cost, -bottleneck\n" + "under PREFER_HIGHER_PRIORITY, hops, EdgeId sequence). Validated in this order before any work\n" + "(ValueError): the roots (non-empty, ids in range), the targets (the same), a node in both sets,\n" + "multipath = False without targets, the residual (length, then every entry in [0, capacity_units]),\n" + "min_residual >= 1, a foreign Eligibility. Excluded roots and targets are skipped; all roots\n" + "excluded gives an empty result."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/trace.hpp b/bindings/python/trace.hpp new file mode 100644 index 0000000..33eef25 --- /dev/null +++ b/bindings/python/trace.hpp @@ -0,0 +1,54 @@ +/* + The trace of a traced placement as Python values (the private hooks only). +*/ +#pragma once + +#include +#include +#include + +#include "netgraph/core/detail/hooks_flag.hpp" + +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include + +#include "netgraph/core/detail/trace.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// A trace value as Python objects (map keys that are decimal integers become ints: the maps +// keyed by source NodeId). +inline py::object trace_obj(const detail::Trace& t) { + using K = detail::Trace::Kind; + switch (t.kind) { + case K::Null: return py::none(); + case K::Bool: return py::bool_(t.b); + case K::Int: return py::int_(t.i); + case K::Str: return py::str(t.s); + case K::List: { + py::list out; + for (const auto& x : t.items) out.append(trace_obj(x)); + return out; + } + case K::Map: { + py::dict out; + for (const auto& [k, x] : t.fields) { + const bool digits = !k.empty() && std::all_of(k.begin(), k.end(), [](char ch) { return ch >= '0' && ch <= '9'; }); + if (digits) { + out[py::int_(std::stoll(k))] = trace_obj(x); + } else { + out[py::str(k)] = trace_obj(x); + } + } + return out; + } + } + return py::none(); +} + +} // namespace netgraph::core::bindings + +#endif diff --git a/bindings/python/units.cpp b/bindings/python/units.cpp new file mode 100644 index 0000000..251de4b --- /dev/null +++ b/bindings/python/units.cpp @@ -0,0 +1,108 @@ +// to_units and as_float: physical values to integer units and back. +#include +#include +#include + +#include +#include +#include + +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// Physical numbers as a float64 array (or a 0-d array for a scalar); bools and strings rejected. +py::array_t physical_array(const py::handle& x) { + if (is_bool(x) || py::isinstance(x) || py::isinstance(x)) { + throw py::value_error("to_units takes physical numbers (a float or an array of floats)"); + } + auto np = py::module_::import("numpy"); + py::array raw = np.attr("asarray")(x); + const char kind = raw.dtype().kind(); + if (kind != 'f' && kind != 'i' && kind != 'u') { + throw py::value_error("to_units takes physical numbers (got dtype " + std::string(py::str(raw.dtype())) + ")"); + } + return py::array_t::ensure(raw); +} + +// x * 2^k for an unbounded Python int, correctly rounded: Python's int shift and int true +// division are exact or correctly rounded, so only a result beyond the finite doubles fails. +double scaled_python_int(const py::object& x, int k) { + try { + py::object r; + if (k >= 0) { + r = py::float_(x.attr("__lshift__")(py::int_(k))); + } else { + r = x.attr("__truediv__")(py::int_(1).attr("__lshift__")(py::int_(-k))); + } + return r.cast(); + } catch (py::error_already_set& e) { + if (e.matches(PyExc_OverflowError)) { + throw py::value_error("as_float: " + std::string(py::str(x)) + + " units times the resolution exceed the largest finite double"); + } + throw; + } +} + +// One Amount to a physical double; the resolution's exponent k is already validated. +double amount_as_float(const py::handle& item, int k, double resolution) { + py::object idx = python_int(item, "as_float: x"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow == 0) return as_float(static_cast(v), resolution); + return scaled_python_int(idx, k); +} + +} // namespace + +void register_units(py::module_& m) { + m.def( + "to_units", + [](const py::object& x, double resolution) -> py::object { + (void)resolution_exponent(resolution); // before the shape: an empty array is no exemption + auto arr = physical_array(x); + if (arr.ndim() == 0) return py::int_(to_units(*arr.data(), resolution)); + if (arr.ndim() != 1) throw py::value_error("to_units takes a float or a 1-D array of floats"); + py::array_t out(arr.size()); + auto in = arr.unchecked<1>(); + auto o = out.mutable_unchecked<1>(); + for (py::ssize_t i = 0; i < in.shape(0); ++i) o(i) = to_units(in(i), resolution); + return std::move(out); + }, + py::arg("x"), py::arg("resolution"), + "Physical value(s) to units of the resolution q, rounding half up: an int for a scalar, an\n" + "int64 array for an array. NaN, infinity, negative values and results at or above 2^62\n" + "units raise ValueError; q must be a power of two in [2^-32, 2^32]."); + + m.def( + "as_float", + [](const py::object& x, double resolution) -> py::object { + const int k = resolution_exponent(resolution); // the resolution before the shape + // A sequence, a NumPy array of any rank included: wide_integers refuses a rank other than 1. + if (!not_a_sequence(x)) { + const auto v = wide_integers(x, "as_float: x"); + py::array_t out(static_cast(v.size())); + auto o = out.mutable_unchecked<1>(); + for (std::size_t i = 0; i < v.size(); ++i) { + o(static_cast(i)) = v.sign(i) == 0 ? as_float(v.value[i], resolution) + : amount_as_float(v.source[py::int_(i)], k, resolution); + } + return std::move(out); + } + return py::float_(amount_as_float(x, k, resolution)); + }, + py::arg("x"), py::arg("resolution"), + "Units to physical value(s): x * q, correctly rounded (exact below 2^53 units). An int, of\n" + "any size, gives a float, an integer array a float64 array; a result beyond the finite\n" + "doubles raises ValueError; q must be a power of two in [2^-32, 2^32]."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/values.hpp b/bindings/python/values.hpp new file mode 100644 index 0000000..7ee09f0 --- /dev/null +++ b/bindings/python/values.hpp @@ -0,0 +1,111 @@ +/* + Small conversions shared by the registration files: read-only NumPy views that + keep their owner alive, the resolution's check, exact counters, int64 arrays, FlowIds, + (EdgeId, units) pairs and the type checks of option arguments. +*/ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/flow_record.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/types.hpp" +#include "integers.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// Read-only NumPy view of a span owned by `owner`. +template +py::array readonly_view(std::span s, const py::handle& owner, const py::dtype& dtype) { + static constexpr T empty_element {}; + py::array arr(dtype, {static_cast(s.size())}, {static_cast(sizeof(T))}, + s.empty() ? &empty_element : s.data(), owner); + py::detail::array_proxy(arr.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + return arr; +} + +template +py::array readonly_view(std::span s, const py::handle& owner) { + return readonly_view(s, owner, py::dtype::of()); +} + +// Validates a resolution (a power of two in [2^-32, 2^32]) and returns its exponent. +inline int resolution_exponent(double resolution) { + (void)as_float(0, resolution); // throws the resolution message + return std::ilogb(resolution); +} + +inline py::object counter_int(const Counter128& c) { + py::int_ hi(c.hi); + py::int_ lo(c.lo); + return hi.attr("__lshift__")(py::int_(64)).attr("__or__")(lo); +} + +inline py::array_t int64_array(const std::vector& v) { + py::array_t out(static_cast(v.size())); + std::copy(v.begin(), v.end(), out.mutable_data()); + return out; +} + +inline FlowId flow_id_of(const py::object& obj) { + if (!py::isinstance(obj) || py::len(obj) != 2) { + throw py::value_error("a FlowId is a tuple (ledger uid, id)"); + } + auto t = py::reinterpret_borrow(obj); + py::object uid = python_int(t[0], "FlowId ledger uid"); + py::object id = python_int(t[1], "FlowId id"); + int o1 = 0, o2 = 0; + const unsigned long long u = PyLong_AsUnsignedLongLong(uid.ptr()); + if (PyErr_Occurred()) { + // Only the range error means no such ledger; another failure (memory running out while that + // error is raised) propagates as raised. + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) throw py::error_already_set(); + PyErr_Clear(); + o1 = 1; + } + const long long i = PyLong_AsLongLongAndOverflow(id.ptr(), &o2); + if (o1 != 0 || o2 != 0 || i < 0) { + throw py::value_error("FlowId (" + std::string(py::str(uid)) + ", " + std::string(py::str(id)) + ") is retired or unknown"); + } + return FlowId {static_cast(u), static_cast(i)}; +} + +inline py::list pairs_list(const std::vector>& v) { + py::list out; + for (const auto& [e, x] : v) out.append(py::make_tuple(e, x)); + return out; +} + +inline OutputOptions outputs_of(const py::object& obj) { + if (obj.is_none()) return OutputOptions {}; + if (!py::isinstance(obj)) { + throw py::type_error("outputs must be an OutputOptions (got " + std::string(py::str(py::type::of(obj).attr("__name__"))) + + ")"); + } + return obj.cast(); +} + +template +Options options_of(const py::object& obj, const char* name) { + if (!py::isinstance(obj)) { + throw py::type_error(std::string(name) + " expected (got " + std::string(py::str(py::type::of(obj).attr("__name__"))) + + ")"); + } + return obj.cast(); +} + +} // namespace netgraph::core::bindings diff --git a/dev/benchmark_profiling_overhead.py b/dev/benchmark_profiling_overhead.py deleted file mode 100644 index 1a32e68..0000000 --- a/dev/benchmark_profiling_overhead.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark for measuring netgraph_core performance and profiling overhead. - -Runs SPF and flow placement operations on a mesh graph topology. -Useful for: - - Measuring profiling infrastructure overhead (compare with/without NGRAPH_CORE_PROFILE=1) - - Regression testing after code changes - - Comparing performance across versions - -Usage: - # Default run (400-node mesh, 500 SPF + 100 flow ops) - python dev/benchmark_profiling_overhead.py - - # Quick smoke test - python dev/benchmark_profiling_overhead.py --quick - - # Custom parameters - python dev/benchmark_profiling_overhead.py --mesh-size 30 --spf-iters 1000 --flow-iters 200 - - # Compare profiling overhead - python dev/benchmark_profiling_overhead.py # baseline - NGRAPH_CORE_PROFILE=1 python dev/benchmark_profiling_overhead.py # with profiling - - # JSON output for scripting - python dev/benchmark_profiling_overhead.py --json -""" - -import argparse -import json -import statistics -import sys -import time - -import numpy as np - -import netgraph_core as ngc - - -def create_mesh_graph(n: int) -> ngc.StrictMultiDiGraph: - """Create an n x n mesh graph with bidirectional edges. - - Args: - n: Grid dimension (creates n*n nodes) - - Returns: - Graph with 4-connected mesh topology, unit costs, 100.0 capacity per edge. - """ - num_nodes = n * n - edges = [] - - for row in range(n): - for col in range(n): - node = row * n + col - if col < n - 1: - right = row * n + (col + 1) - edges.append((node, right, 1, 100.0)) - edges.append((right, node, 1, 100.0)) - if row < n - 1: - down = (row + 1) * n + col - edges.append((node, down, 1, 100.0)) - edges.append((down, node, 1, 100.0)) - - src = np.array([e[0] for e in edges], dtype=np.int32) - dst = np.array([e[1] for e in edges], dtype=np.int32) - cost = np.array([e[2] for e in edges], dtype=np.int64) - cap = np.array([e[3] for e in edges], dtype=np.float64) - ext_edge_ids = np.arange(len(edges), dtype=np.int64) - - return ngc.StrictMultiDiGraph.from_arrays( - num_nodes, src, dst, cap, cost, ext_edge_ids - ) - - -def benchmark_spf( - graph_handle, algs: ngc.Algorithms, num_nodes: int, iterations: int -) -> list[float]: - """Run SPF iterations and return per-call times in ms.""" - times = [] - edge_sel = ngc.EdgeSelection( - multi_edge=False, - require_capacity=False, - tie_break=ngc.EdgeTieBreak.DETERMINISTIC, - ) - - for i in range(iterations): - src = i % num_nodes - start = time.perf_counter() - algs.spf(graph_handle, src, selection=edge_sel) - times.append((time.perf_counter() - start) * 1000) - - return times - - -def benchmark_flow_placement( - g: ngc.StrictMultiDiGraph, graph_handle, algs: ngc.Algorithms, iterations: int -) -> list[float]: - """Run flow placement iterations and return per-call times in ms.""" - num_nodes = g.num_nodes() - times = [] - - for i in range(iterations): - fg = ngc.FlowGraph(g) - cfg = ngc.FlowPolicyConfig() - cfg.flow_placement = ngc.FlowPlacement.EQUAL_BALANCED - cfg.max_flow_count = 16 - cfg.min_flow_count = 16 - cfg.selection = ngc.EdgeSelection( - multi_edge=False, - require_capacity=True, - tie_break=ngc.EdgeTieBreak.PREFER_HIGHER_RESIDUAL, - ) - cfg.require_capacity = True - cfg.multipath = False - cfg.reoptimize_flows_on_each_placement = True - policy = ngc.FlowPolicy(algs, graph_handle, cfg) - - src = i % num_nodes - dst = (num_nodes - 1 - i) % num_nodes - if src == dst: - dst = (dst + 1) % num_nodes - - start = time.perf_counter() - policy.place_demand(fg, src, dst, 0, 1000.0) - times.append((time.perf_counter() - start) * 1000) - - return times - - -def compute_stats(name: str, times: list[float]) -> dict: - """Compute statistics for a benchmark run.""" - return { - "name": name, - "iterations": len(times), - "mean_ms": statistics.mean(times), - "median_ms": statistics.median(times), - "stdev_ms": statistics.stdev(times) if len(times) > 1 else 0, - "min_ms": min(times), - "max_ms": max(times), - "total_ms": sum(times), - } - - -def print_stats(stats: dict) -> None: - """Print benchmark statistics.""" - print(f"\n{stats['name']}:") - print(f" Iterations: {stats['iterations']}") - print(f" Mean: {stats['mean_ms']:.3f} ms") - print(f" Median: {stats['median_ms']:.3f} ms") - print(f" Stdev: {stats['stdev_ms']:.3f} ms") - print(f" Min: {stats['min_ms']:.3f} ms") - print(f" Max: {stats['max_ms']:.3f} ms") - print(f" Total: {stats['total_ms']:.1f} ms") - - -def main(): - parser = argparse.ArgumentParser( - description="Benchmark netgraph_core performance and profiling overhead.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - %(prog)s # default benchmark - %(prog)s --quick # fast smoke test - %(prog)s --mesh-size 30 # larger graph - %(prog)s --json # machine-readable output - NGRAPH_CORE_PROFILE=1 %(prog)s # measure with profiling enabled -""", - ) - parser.add_argument( - "--mesh-size", - type=int, - default=20, - metavar="N", - help="mesh grid size (N x N nodes, default: 20)", - ) - parser.add_argument( - "--spf-iters", - type=int, - default=500, - metavar="N", - help="number of SPF iterations (default: 500)", - ) - parser.add_argument( - "--flow-iters", - type=int, - default=100, - metavar="N", - help="number of flow placement iterations (default: 100)", - ) - parser.add_argument( - "--quick", - action="store_true", - help="quick run with minimal iterations (mesh=10, spf=50, flow=10)", - ) - parser.add_argument( - "--json", - action="store_true", - help="output results as JSON (for scripting)", - ) - args = parser.parse_args() - - if args.quick: - args.mesh_size = 10 - args.spf_iters = 50 - args.flow_iters = 10 - - profiling_enabled = getattr(ngc, "profiling_enabled", lambda: None)() - - if not args.json: - print("=" * 60) - print("NetGraph-Core Benchmark") - print("=" * 60) - print(f"\nVersion: {ngc.__version__}") - if profiling_enabled is not None: - print(f"Profiling: {'ON' if profiling_enabled else 'OFF'}") - print( - f"\nGraph: {args.mesh_size}x{args.mesh_size} mesh ({args.mesh_size**2} nodes)" - ) - print(f"SPF iterations: {args.spf_iters}") - print(f"Flow iterations: {args.flow_iters}") - print("\nCreating graph...", end=" ", flush=True) - - g = create_mesh_graph(args.mesh_size) - num_nodes = g.num_nodes() - num_edges = g.num_edges() - - if not args.json: - print(f"done ({num_nodes} nodes, {num_edges} edges)") - - backend = ngc.Backend.cpu() - algs = ngc.Algorithms(backend) - graph_handle = algs.build_graph(g) - - # Warmup (results discarded, profiling stats reset after) - if not args.json: - print("\nWarmup...", end=" ", flush=True) - sys.stdout.flush() - _ = benchmark_spf(graph_handle, algs, num_nodes, min(10, args.spf_iters)) - _ = benchmark_flow_placement(g, graph_handle, algs, min(5, args.flow_iters)) - if hasattr(ngc, "profiling_reset"): - ngc.profiling_reset() - if not args.json: - print("done", flush=True) - print("\n" + "-" * 40) - print("Running benchmarks...", flush=True) - sys.stdout.flush() - - spf_times = benchmark_spf(graph_handle, algs, num_nodes, args.spf_iters) - spf_stats = compute_stats("Shortest Paths (SPF)", spf_times) - - flow_times = benchmark_flow_placement(g, graph_handle, algs, args.flow_iters) - flow_stats = compute_stats("Flow Placement", flow_times) - - total_ops = args.spf_iters + args.flow_iters - total_time_ms = spf_stats["total_ms"] + flow_stats["total_ms"] - throughput = total_ops / (total_time_ms / 1000) if total_time_ms > 0 else 0 - - if args.json: - result = { - "version": ngc.__version__, - "profiling_enabled": profiling_enabled, - "config": { - "mesh_size": args.mesh_size, - "num_nodes": num_nodes, - "num_edges": num_edges, - "spf_iterations": args.spf_iters, - "flow_iterations": args.flow_iters, - }, - "spf": {k: v for k, v in spf_stats.items() if k != "name"}, - "flow": {k: v for k, v in flow_stats.items() if k != "name"}, - "summary": { - "total_ops": total_ops, - "total_time_ms": total_time_ms, - "throughput_ops_per_sec": throughput, - }, - } - print(json.dumps(result, indent=2)) - else: - print_stats(spf_stats) - print_stats(flow_stats) - - print("\n" + "=" * 60) - print("SUMMARY") - print("=" * 60) - print(f"Total operations: {total_ops}") - print(f"Total time: {total_time_ms:.1f} ms ({total_time_ms / 1000:.2f} s)") - print(f"Throughput: {throughput:.1f} ops/sec") - - if profiling_enabled: - print("\n" + "-" * 40) - print("Profiling stats:", flush=True) - sys.stdout.flush() - ngc.profiling_dump() - sys.stderr.flush() - - -if __name__ == "__main__": - main() diff --git a/dev/coverage_summary.py b/dev/coverage_summary.py index ecd1b33..ae18ad9 100644 --- a/dev/coverage_summary.py +++ b/dev/coverage_summary.py @@ -10,46 +10,93 @@ Defaults: PY_XML = build/coverage/coverage-python.xml CPP_XML = build/coverage/coverage-cpp.xml + +A missing, unreadable or malformed report is an error (exit 1), never zero coverage, and so is a +report that counts no line, which would otherwise show as 100%: gcovr writes one, and exits 0, +when it reads no profile data (the profiles are not where it looks, or the filters match no +file), and coverage.py when the files it measured hold no statement. A report carries both +branch totals or neither, and covers at most what it counts. Each total is a count written in +decimal digits, optionally with a zero fraction ("12" or "12.0"), and at most 2**63 - 1 (far +beyond any real report); anything else (a sign, a nonzero fraction, an exponent, NaN or +infinity) is malformed. The counts are read and compared exactly. """ from __future__ import annotations +import argparse import html import os +import re import sys import xml.etree.ElementTree as ET from typing import Optional, Tuple +# A count: decimal digits, optionally with a zero fraction (some tools write "12.0"). +_COUNT = re.compile(r"([0-9]+)(?:\.0*)?") +_MAX_COUNT = 2**63 - 1 + + +class ReportError(Exception): + """A coverage report is missing, unreadable or malformed.""" + def parse_cobertura_totals( xml_path: str, ) -> Tuple[int, int, Optional[int], Optional[int]]: """Return (lines_valid, lines_covered, branches_valid, branches_covered). - Missing values are returned as None. If the file is missing, returns zeros/None. + The line totals are required; the branch totals are both None when the report has none. + Raises ReportError when the file is missing or unreadable, is not a Cobertura report, + holds a total that is not a count (the module docstring's spelling and bound), has one + branch total without the other, covers more lines or branches than it counts, or counts + no line (it measured nothing). """ - if not os.path.exists(xml_path): - return 0, 0, None, None - try: root = ET.parse(xml_path).getroot() - except Exception: - return 0, 0, None, None + except (OSError, ET.ParseError) as exc: + raise ReportError(f"{xml_path}: cannot read the report: {exc}") from exc + if root.tag != "coverage": + raise ReportError(f"{xml_path}: not a Cobertura report (root <{root.tag}>)") - def _to_int(value: Optional[str]) -> Optional[int]: + def _to_int(name: str, required: bool) -> Optional[int]: + value = root.attrib.get(name) if value is None: + if required: + raise ReportError(f"{xml_path}: no {name} total") return None - try: - # Some tools may write floats; coerce safely - return int(float(value)) - except Exception: - return None - - lines_valid = _to_int(root.attrib.get("lines-valid")) or 0 - lines_covered = _to_int(root.attrib.get("lines-covered")) or 0 - branches_valid = _to_int(root.attrib.get("branches-valid")) - branches_covered = _to_int(root.attrib.get("branches-covered")) - + match = _COUNT.fullmatch(value) + if match is None: + raise ReportError( + f"{xml_path}: {name}={value!r} is not a count (a nonnegative integer)" + ) + digits = match.group(1).lstrip("0") or "0" + if len(digits) > len(str(_MAX_COUNT)) or int(digits) > _MAX_COUNT: + raise ReportError( + f"{xml_path}: {name}={value!r} is above the largest count, 2**63 - 1" + ) + return int(digits) + + lines_valid = _to_int("lines-valid", True) + lines_covered = _to_int("lines-covered", True) + branches_valid = _to_int("branches-valid", False) + branches_covered = _to_int("branches-covered", False) + assert lines_valid is not None and lines_covered is not None + if lines_covered > lines_valid: + raise ReportError(f"{xml_path}: {lines_covered} lines covered of {lines_valid}") + if lines_valid == 0: + raise ReportError( + f"{xml_path}: the report counts no line (no coverage data was read, or the" + " measured files hold no statement)" + ) + if branches_valid is None or branches_covered is None: + if branches_valid is not None or branches_covered is not None: + raise ReportError( + f"{xml_path}: branches-valid and branches-covered must both be present or both absent" + ) + elif branches_covered > branches_valid: + raise ReportError( + f"{xml_path}: {branches_covered} branches covered of {branches_valid}" + ) return lines_valid, lines_covered, branches_valid, branches_covered @@ -70,10 +117,8 @@ def format_text_table( total_lines_valid = py_lines_valid + cpp_lines_valid total_lines_cov = py_lines_cov + cpp_lines_cov - # Branch totals only if present in both - have_branches = (py_br_valid is not None and py_br_cov is not None) or ( - cpp_br_valid is not None and cpp_br_cov is not None - ) + # Branch columns when either report has branch totals; a report without them counts 0 of 0. + have_branches = py_br_valid is not None or cpp_br_valid is not None total_br_valid = (py_br_valid or 0) + (cpp_br_valid or 0) if have_branches else 0 total_br_cov = (py_br_cov or 0) + (cpp_br_cov or 0) if have_branches else 0 @@ -108,31 +153,40 @@ def format_html_table(text_table: str, title: str = "Combined Coverage") -> str:

{html.escape(title)}

{pre}
""" -def main() -> int: - py_xml = ( - sys.argv[1] - if len(sys.argv) > 1 and not sys.argv[1].startswith("--") - else "build/coverage/coverage-python.xml" +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Combine the Python and C++ Cobertura reports into one summary." ) - cpp_xml = ( - sys.argv[2] - if len(sys.argv) > 2 and not sys.argv[2].startswith("--") - else "build/coverage/coverage-cpp.xml" + parser.add_argument( + "py_xml", nargs="?", default="build/coverage/coverage-python.xml" ) + parser.add_argument("cpp_xml", nargs="?", default="build/coverage/coverage-cpp.xml") + parser.add_argument( + "--html", metavar="OUT_HTML", help="also write the table as HTML" + ) + args = parser.parse_args(argv) - html_out: Optional[str] = None - for arg in sys.argv[1:]: - if arg.startswith("--html="): - html_out = arg.split("=", 1)[1] - - py_totals = parse_cobertura_totals(py_xml) - cpp_totals = parse_cobertura_totals(cpp_xml) + try: + py_totals = parse_cobertura_totals(args.py_xml) + cpp_totals = parse_cobertura_totals(args.cpp_xml) + except ReportError as exc: + print(f"coverage_summary.py: {exc}", file=sys.stderr) + return 1 text = format_text_table(py_totals, cpp_totals) print(text) + html_out: Optional[str] = args.html if html_out: - os.makedirs(os.path.dirname(html_out), exist_ok=True) - with open(html_out, "w", encoding="utf-8") as f: - f.write(format_html_table(text)) + out_dir = os.path.dirname(html_out) + try: + if out_dir: + os.makedirs(out_dir, exist_ok=True) + with open(html_out, "w", encoding="utf-8") as f: + f.write(format_html_table(text)) + except OSError as exc: + print( + f"coverage_summary.py: cannot write {html_out}: {exc}", file=sys.stderr + ) + return 1 return 0 diff --git a/dev/cpp-tests.sh b/dev/cpp-tests.sh index aab39be..82ea132 100644 --- a/dev/cpp-tests.sh +++ b/dev/cpp-tests.sh @@ -1,27 +1,108 @@ #!/usr/bin/env bash -# Propagate configuration, build, and test failures; bound local CPU use. +# The C++ test runner (make cpp-test, cpp-test-off, sanitize-test, tsan-test, check-ci, cov). +# +# bash dev/cpp-tests.sh MODE +# +# MODE private hooks build type build directory +# release ON Release (LTO) build/cpp-tests +# off OFF Release (LTO) build/cpp-tests-off the shipped configuration +# sanitize ON Debug, ASan + UBSan build/cpp-sanitize +# tsan ON Debug, TSan build/cpp-tsan +# coverage ON Debug, --coverage build/cpp-tests-cov used by make cov +# +# Each mode has its own build directory and states every project option on the command line, so a +# value cached by an earlier configure never decides this one. One generator (Ninja), parallelism +# bounded by CMAKE_BUILD_PARALLEL_LEVEL (default 4), ctest fails when it finds no test, and every +# configuration, build or test failure fails the script. The skipped tests are checked: none in a +# hooks-on mode, exactly EXPECTED_OFF_SKIPS (the tests that need a hook) in mode off, so a test that +# silently became hooks-only is caught. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# The C++ tests that need the private hooks and skip without them (GTEST_SKIP). Inside tests that +# still run, three checks are weaker without them: the helper check_deletion +# (flow_core_deletion_tests.cpp) checks each re-solve's final result, but observes each repair step +# only through the probe hook (see that file's header); PolicyFixtures.EveryFixtureReplaysIdentically +# compares no trace and replays only the 58 fixtures that keep the default creation cap, and +# PolicyFixtures.ACorruptedExpectationFails corrupts only what that replay compares, in those 58 +# fixtures (see flow_policy_fixture_tests.cpp's header). +EXPECTED_OFF_SKIPS=" +FlowCoreDeletion.CheckpointsEqualTheIterativeUpdate +FlowCoreDeletion.RerunningTheFailedSearchAddsExactlyOneSearch +GraphLayout.CombinedKeyPassesOnlyBelowNTimesKEqualsE +GraphLayout.HashAndFallbackBoundaryAt65536DistinctCosts +GraphLayout.ProbeBudgetFallbackInCollectionAndGrowth +GraphLayout.ProbeBudgetIsCumulativeAcrossTheLargeTableGrowths +GraphLayout.RadixFallbackDigitsOnBothSidesOfTheCombinedKeyBoundary +GraphLayout.RunsOfEqualCostsSkipTheLookup +Search.QueueDispatchAtTheWidthLimit +" + +die() { echo "dev/cpp-tests.sh: $*" >&2; exit 1; } + jobs=${CMAKE_BUILD_PARALLEL_LEVEL:-4} case "$jobs" in ''|*[!0-9]*|0) echo 'CMAKE_BUILD_PARALLEL_LEVEL must be positive.' >&2; exit 2 ;; esac -case "${1:-release}" in - release) build_dir=build/cpp-tests; build_type=Release; sanitize=OFF ;; +mode=${1:-} +hooks=ON; sanitize=OFF; tsan=OFF; coverage=OFF +case "$mode" in + release) build_dir=build/cpp-tests; build_type=Release ;; + off) build_dir=build/cpp-tests-off; build_type=Release; hooks=OFF ;; sanitize) build_dir=build/cpp-sanitize; build_type=Debug; sanitize=ON ;; - *) echo 'Expected release or sanitize.' >&2; exit 2 ;; + # ThreadSanitizer (data races of the worker threads and the shared DagCache); the longest + # instrumented test takes about 30 s, inside the common timeout. + tsan) build_dir=build/cpp-tsan; build_type=Debug; tsan=ON ;; + coverage) build_dir=build/cpp-tests-cov; build_type=Debug; coverage=ON ;; + *) echo 'Usage: bash dev/cpp-tests.sh release|off|sanitize|tsan|coverage' >&2; exit 2 ;; esac + +command -v cmake >/dev/null 2>&1 || die "cmake not found on PATH (make dev installs it into venv/bin)" +command -v ctest >/dev/null 2>&1 || die "ctest not found on PATH (make dev installs it into venv/bin)" +command -v ninja >/dev/null 2>&1 || die "ninja not found on PATH (make dev installs it into venv/bin)" +cache="$build_dir/CMakeCache.txt" +if [[ -f "$cache" ]] && ! grep -qx 'CMAKE_GENERATOR:INTERNAL=Ninja' "$cache"; then + die "$build_dir was configured by another generator ($(sed -n 's/^CMAKE_GENERATOR:INTERNAL=//p' "$cache")); remove that directory and rerun" +fi + if [[ "$(uname -s)" == Darwin ]]; then - export CC="$(xcrun --find clang)" - export CXX="$(xcrun --find clang++)" + CC=$(xcrun --find clang) + CXX=$(xcrun --find clang++) + export CC CXX export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-15.0}" leak_detection=0 else leak_detection=1 fi -cmake -S . -B "$build_dir" -DNETGRAPH_CORE_BUILD_TESTS=ON \ - -DNETGRAPH_CORE_SANITIZE="$sanitize" -DCMAKE_BUILD_TYPE="$build_type" -cmake --build "$build_dir" --config "$build_type" --parallel "$jobs" if [[ "$sanitize" == ON ]]; then - export ASAN_OPTIONS="${ASAN_OPTIONS:-detect_leaks=$leak_detection}" + # A sanitizer report must fail, including during test discovery at build time. Append + # the failure settings after the caller's options; keep the platform's leak policy. + export ASAN_OPTIONS="${ASAN_OPTIONS:-detect_leaks=$leak_detection}:halt_on_error=1:exitcode=1" + export UBSAN_OPTIONS="${UBSAN_OPTIONS:+$UBSAN_OPTIONS:}halt_on_error=1:exitcode=1" +fi +if [[ "$tsan" == ON ]]; then + export TSAN_OPTIONS="${TSAN_OPTIONS:-second_deadlock_stack=1}:halt_on_error=1:exitcode=1" +fi +cmake -S . -B "$build_dir" -G Ninja -DCMAKE_BUILD_TYPE="$build_type" \ + -DNETGRAPH_CORE_BUILD_TESTS=ON -DNETGRAPH_CORE_PRIVATE_HOOKS="$hooks" \ + -DNETGRAPH_CORE_SANITIZE="$sanitize" -DNETGRAPH_CORE_TSAN="$tsan" \ + -DNETGRAPH_CORE_COVERAGE="$coverage" -DNETGRAPH_CORE_NATIVE=OFF +cmake --build "$build_dir" --parallel "$jobs" +if [[ "$coverage" == ON ]]; then + # Test discovery runs each binary at build time and writes profiles that are no test's coverage. + find "$build_dir" -name '*.gcda' -delete +fi +log="$build_dir/ctest.log" +ctest --test-dir "$build_dir" --output-on-failure --parallel "$jobs" --timeout 120 --no-tests=error \ + 2>&1 | tee "$log" + +# ctest reports a GTEST_SKIP as "***Skipped" (gtest_discover_tests' skip pattern). +skipped=$(sed -n 's/^ *[0-9]*\/[0-9]* Test *#[0-9]*: \([^ ]*\) .*\*\*\*Skipped.*/\1/p' "$log" | LC_ALL=C sort) +if [[ "$hooks" == OFF ]]; then + expected=$(echo "$EXPECTED_OFF_SKIPS" | sed '/^$/d' | LC_ALL=C sort) + if [[ "$skipped" != "$expected" ]]; then + diff <(echo "$expected") <(echo "$skipped") >&2 || true + die "the skipped tests differ from the expected hook-only tests (< expected, > skipped)" + fi + echo "hooks off: exactly the $(echo "$expected" | wc -l | tr -d ' ') expected hook-only tests skipped" +elif [[ -n "$skipped" ]]; then + die "tests skipped in a hooks-on build: $(echo "$skipped" | tr '\n' ' ')" fi -ctest --test-dir "$build_dir" --build-config "$build_type" \ - --output-on-failure --parallel "$jobs" --timeout 120 --no-tests=error diff --git a/dev/perf/README.md b/dev/perf/README.md index d9c6cc0..32024df 100644 --- a/dev/perf/README.md +++ b/dev/perf/README.md @@ -1,21 +1,380 @@ -# Performance tooling - -Developer-only tools for measuring NetGraph-Core. Nothing here ships in the -wheel or the sdist, and nothing here is run by CI; the repository gates -(`make check-ci`, `make sanitize-test`) remain the acceptance tests. - -- [`spf_queue/`](spf_queue/): the shortest-path frontier queue and workspace - work (plan, parity harness, old-versus-new bit-gate, NetGraph-level - benchmark, recorded A/B/A results and the independent review). Start with - [`spf_queue/PLAN.md`](spf_queue/PLAN.md). -- [`../benchmark_profiling_overhead.py`](../benchmark_profiling_overhead.py): - SPF and placement micro-benchmark used for the PGO training run - (`make install-pgo`) and for measuring profiling overhead. - -Conventions for performance claims (see `AGENTS.md`): verify correctness -first (bit-identical outputs where the contract is exact, certificates for -flows), then measure old/new/old in fresh processes on one quiet machine, -record load, hashes and raw samples, and treat differences inside the -run-to-run band as inconclusive. The research record behind the queue work, -including the Apple GPU investigation that preceded it, is in -[`../research/`](../research/). +# Timing tool (A/B/A) + +Developer-only tool for timing two builds of NetGraph-Core against each other, as `AGENTS.md` +asks for every performance claim. Nothing here ships in the wheel or the sdist, and no timing +runs in CI; only the tool's own tests do (below), and the repository gates (`make check-ci`, +`make sanitize-test`, `make tsan-test`) stay the acceptance tests. Verify correctness first +(bit-identical results where the contract is exact, certificates for flows); time second. + +**A/B/A** means: the reference build A, the candidate build B, then A again, each block in a +fresh process, repeated seven times; the candidate's time is compared with the mean of the two +reference blocks around it, so slow drift of the machine cancels out. A bootstrap interval over +the seven triples says whether a difference is real. + +| File | Role | +| --- | --- | +| `build.sh` | builds one harness binary against the Core of a revision or of the working tree, with its build record | +| `harness.cpp`, `aba_rows.hpp`, `aba_api.hpp` | the harness: the rows (workloads, semantic work per iteration, digests) and the adapter that maps them onto the current public Core API (plus two private functions of the queue rule, below) | +| `aba.py` | runs one A/B/A comparison: blocks, statistics, verdicts, `report.json`, `summary.md`, `blocks.csv`, `meta.txt` | +| `queries/` | the frozen row inputs (`*.json`, `storage/inputs.json`) with their `SHA256SUMS` | +| `test_tool.py` | the tool's own tests (`make perf-tool-test`) | + +## Quick start + +Compare the working tree (candidate) with a revision (reference); both binaries carry the same +rows, each compiled against its own Core. Run on a quiet machine, nothing else running. + +```sh +bash dev/perf/build.sh build/perf/aba/ref # e.g. HEAD, or main +bash dev/perf/build.sh worktree build/perf/aba/cand +venv/bin/python dev/perf/aba.py \ + --old-bin build/perf/aba/ref/harness --new-bin build/perf/aba/cand/harness \ + --old-label ref --new-label cand --rows "kernel.*,maxflow.*" --out build/perf/aba/run +``` + +`aba.py` needs Python 3.10 or newer (the venv's; macOS's system `python3` 3.9 is refused with a +message). The run prints a plan (the expected duration of every row) before the first timed +block, then one line per row, and writes `report.json`, `summary.md` (the table to read), +`blocks.csv` (every block; in an aborted run, those of the finished warm-up pairs and triples) +and `meta.txt` (every hash) to `--out` (default `dev/perf/results//`, ignored by +git). The run's verdict is the worst row verdict, and a row whose results differ between the two +builds fails: the tool compares two builds of the current Core, so a change it times (an +optimisation) must leave every result identical. A change that is meant to change results is +checked for correctness by the test suites, not timed against the old results. The adapter +`aba_api.hpp` is written against the current public API, apart from the queue check of the +storage search rows, which reads the library's queue rule through the private +`detail::bucket_width` and `detail::uses_buckets`: a reference revision with another API needs +the adapter adapted first. + +**Exit statuses and acceptance.** `aba.py` exits 0 when the run completed, whatever its verdict +and conformance; 1 when it aborted after starting blocks; 2 when it refused an input or failed +before the first block; 130 for Ctrl-C before the first block. Every failure writes its reason +to `report.json` and `summary.md`, with verdict `aborted` and conformance false. A reason names +the side, mode and row when it comes from a block. A refusal before row selection writes a +minimal report with no rows. A failed harness, malformed response, changed or unreadable binary, +non-finite derived value, operating-system error, internal exception or Ctrl-C ends the run. +The first Ctrl-C stops the active harness; later interrupts are held while the outputs are +written. An interrupt during final writing records an abort and rewrites both outputs. + +Both files are created before the first block. Each final output is attempted even if another +fails; text that cannot be formed is replaced by a minimal abort record. Complete output text +replaces its file in one operation; if directory permissions no longer allow replacement, an +existing writable file is retried directly with the abort reason. An invalid, ambiguous, +occupied or unwritable `--out` cannot receive a new report; existing unrelated files are never +overwritten. A persistent write failure (for example a full disk), or a process killed by an +uncatchable signal, can prevent output. Write failures are also reported on stderr. `--help` +prints help and writes no run files. Both outputs list every selected row with the step it reached: `measured`, and in an +aborted run `not run`, `warm-up` (its warm-up step did not finish: a warm-up block failed, or a +count or the expected duration derived from them is not finite), `planned` (warmed up, not +timed) or `timing` (its timed step did not finish: a timed block failed, or a row statistic +derived from them is not finite). A run killed by another signal writes nothing more. Its exit +status is no acceptance: a run is accepted when it is conforming and its verdict is `pass`, the +first two items of `summary.md` (and `report.json`'s `conforming` and `verdict`). + +## Building the two sides + +`build.sh` requires `bash`, `git`, `tar`, `python3` and the compiler on `PATH` (on macOS, +`xcrun` for an explicitly selected compiler). `vtool` supplies optional build diagnostics. + +`bash dev/perf/build.sh REV OUT_DIR` compiles this checkout's `harness.cpp` (with `aba_rows.hpp` +and `aba_api.hpp`) against the Core sources of `REV`: `worktree` takes this checkout's `include/` +and `src/` with their uncommitted changes; a git revision is extracted with `git archive` into +`OUT_DIR/src`. `REV` is required (there is no default reference), and a revision git does not +know is refused. `OUT_DIR` must be absent or empty; it receives `harness` and the build record +`harness.json`. The compiler is `clang++` from `PATH` with the first `flags=(...)` line of +`build.sh` (`-std=c++20 -O3 -DNDEBUG -funroll-loops -fno-math-errno -fno-trapping-math -Wall +-Wextra`). `CXX` overrides the compiler; on macOS a set `CXX` also gets `-isysroot "$(xcrun +--show-sdk-path)"` (a toolchain compiler such as `$(xcrun --find clang++)` finds no system headers +without it), and a set `CXX` without an SDK is refused. A refusal exits 2. A failed compile exits +with the compiler's status, and a failure while the build record is written exits non-zero; +either leaves neither `harness` nor `harness.json` (a revision's extracted sources stay in +`OUT_DIR/src`, so a second attempt needs another or an emptied `OUT_DIR`). + +### Build records + +`harness.json` names everything that changes the binary's bytes: + +| Field | Meaning | +| --- | --- | +| `binary`, `binary_sha256` | the output path and the hash of the file written there; the file name is part of the bytes (below) | +| `rev`, `commit`, `dirty_core_paths` | the revision asked for, its commit (`HEAD` for `worktree`), and for `worktree` the number of paths `git status --short` lists under `include/` and `src/` (0 for a revision) | +| `core_sources_sha256` | sha256 of `" \n"` lines over every file under the `include/` and `src/` compiled against (the checkout's or the extracted ones), sorted by path; a path with a part that starts with a dot is left out | +| `driver_sha256` | the sha256 of `dev/perf/harness.cpp` and of every `dev/perf/*.hpp`, by path | +| `compiler`, `compiler_path`, `cxx_env` | the compiler's `--version` line, the program the name resolved to on `PATH`, and `CXX` as given (null when unset) | +| `flags`, `isysroot` | the flags passed (`-isysroot ` included when added), and that SDK path (null when none was passed) | +| `sdk.path`, `sdk.version`, `sdk.sdkroot_env`, `sdk.source` | the SDK `xcrun` names (the one the `clang++` shim and `-isysroot` use) and `SDKROOT`, which overrides the shim's choice; path and version null when `xcrun` names no SDK | +| `deployment_target.requested` | `MACOSX_DEPLOYMENT_TARGET` (null when unset: the compiler then targets the SDK's own OS version) | +| `deployment_target.effective_minos`, `binary_sdk`, `platform`, `raw`, `source` | the minimum OS and SDK version the binary carries, read back from its `LC_BUILD_VERSION` load command with `vtool -show-build` (the first value of each when the binary carries several; `raw` has every line) | +| `built_by` | `dev/perf/build.sh` | + +Besides the sources and the flags, the bytes depend on the **deployment target** (the same +sources built with `MACOSX_DEPLOYMENT_TARGET=15.0` and without it differ), the **output file +name** (the linker signs every arm64 binary ad hoc with the file name as the signature's +identifier, and the binary's UUID follows; copying a binary keeps its bytes, the name at link +time is what counts), the **SDK** and the **compiler**. A pinned binary hash is reproducible +only with all of them equal. `aba.py` refuses a binary whose hash differs from its record. Every field in the table is +required and no other key is accepted, including inside `sdk` and `deployment_target`. +All values are strings except `dirty_core_paths` (a finite nonnegative integer), the three +objects `sdk`, `deployment_target`, `driver_sha256`, and these nullable strings: `compiler_path`, `cxx_env`, `isysroot`, `sdk.path`, `sdk.version`, +`sdk.sdkroot_env`, and `deployment_target.requested`, `effective_minos`, `binary_sdk`, `platform`, +`raw`. A null means the value was unavailable or the environment variable was unset. `binary_sha256` and `core_sources_sha256` are 64 lowercase hexadecimal digits; +`driver_sha256` has exactly this checkout's driver paths and matching hashes. Descriptive +strings are recorded as text; the old build path, revision and compiler are not re-executed or +used to locate inputs. + +## Protocol + +**Blocks.** Each block is a fresh process of one side's harness running one row: setup +(untimed), one untimed iteration that fixes the reference digest, then a fixed iteration count +with every iteration timed; the harness reports the median iteration time and `block_s`, its own +wall-clock measurement of the timed loop. Every iteration must return the reference digest (else +the harness exits 3 and the run aborts), which the block line carries as 16 lowercase hexadecimal +digits. A block's whole output is its response, and the run +aborts unless it is exactly one block line for the requested row ended by a newline, with +nothing before or after it (a usable line is never picked out of a broken response). A block +line is malformed, and aborts the run, when it is not one JSON object, names a key twice or a +key outside the listed fields, or a required number is missing, of another type, not finite (an integer beyond the largest +float counts as not finite) or outside its domain: the iteration count is an integer in [1, 2^63 - 1]; the median iteration +time and the harness's own peak RSS are positive; `block_s`, `setup_s` and the children's peak +RSS may be 0 (the harness prints the two times rounded to microseconds, and starts no child +process); the statistics no rule reads (`mean_s`, `min_s`, `p10_s`, `p90_s`, `max_s`) are +nonnegative when present. A timed block must report the iteration count it was given. The +numeric options are checked the same way (`--tolerance`, `--rss-floor-mib` and +`--calibration-margin` may be 0, `--seed` is an integer of either sign within the finite float range, the others are positive), and the +values derived from them must be finite (`--target-seconds` times one plus the margin, the floor +in bytes). "Inputs" below lists every input's rule. + +**Derived values.** Every number of a block line is finite, and every one but the iteration +count is read as a float, but arithmetic on huge ones need not be finite: the mean of two +medians near 1e308 is infinite, and a ratio against it would read 0, a false pass. So a value +derived from the blocks that is not finite (a mean of A1 and A2, a ratio, the statistic, its +interval, the drift, a median, a peak sum, the memory allowance, a calibrated count, an expected +duration or a finish time beyond any date) aborts the run before the row's statistics or verdict +are recorded or printed. + +**Per row.** One warm-up block per side (discarded; it calibrates the iteration counts), then +seven triples A1, B, A2 in that order (fewer only with `--allow-short`, which labels the run +"not gate evidence"). The counts are **per side**: `ceil(target * (1 + margin) / median_side)` +from that side's own warm-up block (a calibrated count above 2^63 - 1 aborts with the side and row named) (`--target-seconds`, default 2, `--calibration-margin`, +default 0.10), so every block on either side lasts about the target even when the two sides' +speeds differ by orders of magnitude. A1 and A2 share side A's count. `--iters` pins the counts +instead (below). The block value is the median iteration time, so the ratio does not depend on +the counts. The warm-up blocks of every selected row run first; the expected duration of every +row and of the run is printed and stored (`report.json` `plan`, each row's `expected`) before +the first timed block. + +**Statistics.** `r_t = B_t / mean(A1_t, A2_t)`; the statistic is `median_t(r_t - 1)`; its 95 +percent percentile interval comes from `--resamples` (default 10,000) bootstrap resamples of +whole triples, seeded with `--seed`; drift is `median_t |A1_t / A2_t - 1|`. + +**Verdicts per row.** + +* timing: `inconclusive` when the drift exceeds the tolerance (`--tolerance`, default 0.03), + else `pass` when the interval's upper bound is within the tolerance, else `fail`; +* blocks: `fail` when any timed block's `block_s` is below `--target-seconds`; +* same-side clusters: per side, the timed blocks' per-iteration times sorted; a gap between two + consecutive blocks above the tolerance with at least two blocks on each side marks the side + `bimodal` and fails the row even when the median passes; +* memory (below); +* digests: every timed block of both sides must report one and the same digest. + +The row is `fail` if any check fails, else `inconclusive` if timing is, else `pass`. The run's +verdict is `aborted` when the run aborted, else the worst row verdict. A run is **conforming** +(gate evidence) only when it did not abort, `--allow-short` was not given (without it fewer than +seven triples or a target below 2 s are refused; a measured block shorter than the target fails +its row), no timed block was short and the +allocator environment is normal; the summary's first line says which. + +**Memory.** Each block's peak RSS (resident set size: the memory the process holds in RAM) of the +harness process tree is the larger of the harness's own `getrusage` (self + children) +`ru_maxrss` and `ru_maxrss` from the orchestrator's `os.wait4` (macOS reports bytes, Linux KiB, +converted). The harness's own record of the iteration times is the same in every block, whatever +its iteration count: allocated and written before the row's setup, it keeps a count per duration +below 65,536 clock ticks (65.5 µs with the nanosecond clocks of macOS and Linux) and a list of +the longer ones with 65,536 slots, which grows only past 4.3 s of such iterations in one block +(1 MiB of bookkeeping in all; its statistics are those of the sorted list of every iteration's +time). A faster side runs more iterations, and a record that grew with them would count in its +peak. Reference = the largest A1 block peak, candidate = the largest B block peak (over the +timed triples); the row fails when the candidate exceeds `max(1.10 * reference, reference + 8 +MiB)` bytes (`--budget`, `--rss-floor-mib`). A1, the first reference block of every triple, is +fixed in advance: a larger A2 peak or a larger warm-up peak never rescues a failure. Why the +maxima: macOS's allocator keeps freed large blocks in a cache whose pages stay resident, so on +several rows the block peaks of any build land on one of two levels about 9 to 10 MiB apart at +random; medians can fall on different levels by chance and fail identical binaries, while the +largest peaks of both sides reach the upper level. A row records `rss_a1_max_bytes`, +`rss_b_max_bytes`, `rss_allowance_bytes`, `rss_balanced_ratio`, `rss_a2_max_bytes` (a +diagnostic) and `rss_verdict`. The check compares the largest observed peaks; it is no absolute +memory bound. + +**Allocator environment.** `aba.py` neither sets nor clears an allocator variable +(`MallocLargeCache=0` would hide the cache levels and with them a candidate's genuine peak). It +records the harness's environment in `report.json` `allocator_env` and on the summary's +"allocator environment" line: whether `MallocLargeCache` is set, and every variable whose name +starts with `Malloc` or `MALLOC_` or is `DYLD_INSERT_LIBRARIES`, `LD_PRELOAD` or +`GLIBC_TUNABLES`. A run with any such variable is not conforming. + +**Provenance.** `meta.txt` lists the sha256 of both harness binaries (each checked against its +build record when it has one) and of the orchestrator and driver sources (`aba.py`, +`harness.cpp`, `aba_rows.hpp`, `aba_api.hpp`, hashed at the start). A side's binary is +re-hashed before and after every block and at the end of the run; a difference, or a binary +that can no longer be read, aborts the run. `--expect-meta meta.txt` refuses to start unless +every hash equals a previous run's (the refusal names the hashes that differ). Both sides' +harnesses run in the repository root with the environment `aba.py` was started in, unchanged: +`aba.py` sets and clears no variable. + +**Machine state.** Each block records the 1-minute load before and after; the summary has a +machine-state line (load averages, top CPU consumers) at the start and the end of the run. + +### The sides + +Every selected row runs as A1 = `--old-bin`, B = `--new-bin`, A2 = `--old-bin` with every rule +above: two builds of the current Core. Both must run this checkout's driver (`build.sh` builds +them so): the two harnesses must list the same rows, record for record, and a side's build +record, when it has one, must carry this checkout's driver as its `driver_sha256`: exactly the +driver files `meta.txt` names (`harness.cpp`, `aba_rows.hpp`, `aba_api.hpp`), each with the +sha256 it has at the start of the run; else the run is refused, so a harness built before the +rows or the adapter changed is not timed. A side without a build record is not checked this way +(`report.json` records its build record as empty). + +### Selecting rows + +`--rows` takes comma-separated globs over row names and groups (`all`, `construct.*`, +`kernel`); spaces around a glob are ignored, and no text means `all`. **A glob that matches no +row or group refuses the run** before any block, with the list of the groups that exist, even +beside globs that match. The report records, per glob, the rows it selected and the rows of its +groups it did not, and per group the rows the whole selection left out. + +Other options: `--iters row=N,...` (both sides) or `row=NA:NB,...` (per side) pins the timed +blocks' iteration counts of a row in place of the calibrated ones (every item names a selected +row, once; a count is at most 2^63 - 1, the largest the harness reads); `--triples` (default 7, +fewer only with `--allow-short`, "Per row" above); `--seed` (default 20261003) and +`--resamples` of the bootstrap; `--old-label` and `--new-label` (defaults `old` and `new`; two +different labels, below); `--out` (absent or empty). + +**Names.** A row's name and its group are 1 to 64 of the characters `a`-`z`, `0`-`9`, `.`, +`_` and `-`; a side's label is 1 to 32 of `a`-`z`, `0`-`9`, `_` and `-` (no dot). They make the +stderr logs' names, `blocks/.