diff --git a/.coveragerc b/.coveragerc index e312baf..4aaecbe 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,8 +3,6 @@ branch = True data_file = build/coverage/.coverage source = netgraph_core -omit = - python/netgraph_core/_docs.py [report] show_missing = True diff --git a/.github/ci-constraints.txt b/.github/ci-constraints.txt new file mode 100644 index 0000000..d0991f1 --- /dev/null +++ b/.github/ci-constraints.txt @@ -0,0 +1,78 @@ +# Exact versions for CI's Python environments and for the coverage venv of `make cov`, so a new +# release of a tool cannot change a CI result without a change in this repository: the accepted +# development environment's versions, and for what only Linux, Windows or Python 3.11 installs, +# the version a resolver chose for those platforms (its marker says where it applies). pip itself +# is one of them: a job installs this pip (`pip install -U -c` this file, which also downgrades a +# newer one) before anything else. They reach a build environment only through the mechanism of +# the tool that fills it: pip's own isolated build takes --build-constraint or +# PIP_BUILD_CONSTRAINT (since pip 26.2 never -c or PIP_CONSTRAINT), `python -m build` takes +# PIP_CONSTRAINT, and its uv installer (the release wheels) UV_CONSTRAINT. NumPy 2.5 needs Python +# 3.12; Python 3.11 gets the last NumPy that supports it. Update deliberately. +build==1.5.0 +certifi==2026.7.22 +cfgv==3.5.0 +charset-normalizer==3.5.1 +cmake==4.4.2 +colorlog==6.12.0 +coverage==7.15.4 +distlib==0.4.3 +docutils==0.23 +filelock==3.32.3 +gcovr==8.6 +id==1.6.1 +identify==2.6.19 +idna==3.19 +iniconfig==2.3.0 +jaraco.classes==3.4.0 +jaraco.context==6.1.2 +jaraco.functools==4.6.0 +Jinja2==3.1.6 +keyring==25.7.0 +lxml==6.1.2 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +more-itertools==11.1.0 +nh3==0.3.6 +ninja==1.13.0 +nodeenv==1.10.0 +packaging==26.3 +pathspec==1.1.1 +pip==26.2.1 +platformdirs==4.11.3 +pluggy==1.6.0 +pre_commit==4.6.2 +pybind11==3.1.0 +pybind11-stubgen==3.0.0 +Pygments==2.21.0 +pyproject_hooks==1.2.0 +pyright==1.1.401 +pytest==9.1.1 +pytest-cov==7.1.0 +python-discovery==1.5.2 +PyYAML==6.0.3 +readme_renderer==45.0 +requests==2.34.2 +requests-toolbelt==1.0.0 +rfc3986==2.0.0 +rich==15.0.0 +ruff==0.11.13 +scikit_build_core==1.0.3 +setuptools==84.0.0 +twine==7.0.0 +typing_extensions==4.16.0 +urllib3==2.7.0 +virtualenv==21.7.4 +wheel==0.48.0 +numpy==2.5.2; python_version >= "3.12" +numpy==2.4.4; python_version < "3.12" +backports.tarfile==1.2.0; python_version < "3.12" +importlib_metadata==9.0.1; python_version < "3.12" +zipp==4.1.1; python_version < "3.12" +cffi==2.1.1; sys_platform == "linux" +cryptography==50.0.2; sys_platform == "linux" +jeepney==0.9.0; sys_platform == "linux" +pycparser==3.11; sys_platform == "linux" +SecretStorage==3.5.0; sys_platform == "linux" +colorama==0.4.6; sys_platform == "win32" +pywin32-ctypes==0.2.3; sys_platform == "win32" diff --git a/.github/workflows/proofs.yml b/.github/workflows/proofs.yml new file mode 100644 index 0000000..8824f22 --- /dev/null +++ b/.github/workflows/proofs.yml @@ -0,0 +1,187 @@ +name: Lean verification + +on: + pull_request: + push: + branches: ["**"] + workflow_dispatch: + workflow_call: + inputs: + sdist-artifact: + description: Existing source artifact to check, rather than building a new archive + type: string + default: "" + wheels-pattern: + description: Existing release wheel artifacts corresponding to that archive + type: string + default: "" + +permissions: + contents: read + +env: + LEAN_NUM_THREADS: "2" + PROOFS_DEPENDENCY_ORIGIN: cache+local + +jobs: + model: + name: Lean proofs + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + # Official action, pinned to its reviewed source. The package's own entry point + # controls build/audit/replay; the action supplies Elan and the pinned mathlib cache. + - uses: leanprover/lean-action@24581d74d674803a25a81b10b059c5c0cdd344c6 + with: + lake-package-directory: proofs + auto-config: "false" + build: "false" + test: "false" + lint: "false" + use-github-cache: "false" + use-mathlib-cache: "true" + - name: Model proofs, fingerprints and fresh kernel replay + run: bash dev/proofs/run.sh replay + - name: Real-discovery negative controls + run: bash dev/proofs/run.sh test + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lean-model-evidence + path: | + build/proofs/check.json + build/proofs/replay.json + build/proofs/*.log + if-no-files-found: error + + distribution: + name: Lean distribution + runs-on: ubuntu-22.04 + env: + PIP_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt + PIP_BUILD_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt + CMAKE_BUILD_PARALLEL_LEVEL: "4" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Pinned packaging tools + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt build + - uses: actions/download-artifact@v4 + if: inputs.sdist-artifact != '' + with: + name: ${{ inputs.sdist-artifact }} + path: build/proofs/dist + - uses: actions/download-artifact@v4 + if: inputs.wheels-pattern != '' + with: + pattern: ${{ inputs.wheels-pattern }} + merge-multiple: true + path: build/proofs/dist + - name: Build candidate distributions with excluded local by-products present + if: inputs.sdist-artifact == '' + run: | + mkdir -p proofs/.lake build/proofs + echo must-not-ship > proofs/.lake/packaging-plant.keep + echo must-not-ship > build/proofs/generated-report.keep + python -m build --sdist --wheel --outdir build/proofs/dist \ + --config-setting=build-dir=build/proofs/wheel-build + - name: Check exact members, record archive identity and extract + shell: bash + run: | + sdists=(build/proofs/dist/*.tar.gz) + wheels=(build/proofs/dist/*.whl) + [[ ${#sdists[@]} -eq 1 && -f ${sdists[0]} ]] + [[ -f ${wheels[0]} ]] + proof_extract="$RUNNER_TEMP/netgraph-core-proofs-sdist" + echo "PROOFS_EXTRACTED=$proof_extract" >> "$GITHUB_ENV" + args=(--sdist "${sdists[0]}" --receipt build/proofs/receipt.json --extract "$proof_extract") + for wheel in "${wheels[@]}"; do args+=(--wheel "$wheel"); done + python -I -B dev/proofs/package.py "${args[@]}" + test ! -e "$proof_extract/.git" + test ! -e "$proof_extract/dev" + test ! -e "$proof_extract/proofs/.lake" + ! git -C "$proof_extract" rev-parse --show-toplevel + - uses: leanprover/lean-action@24581d74d674803a25a81b10b059c5c0cdd344c6 + with: + lake-package-directory: ${{ env.PROOFS_EXTRACTED }}/proofs + auto-config: "false" + build: "false" + test: "false" + lint: "false" + use-github-cache: "false" + use-mathlib-cache: "true" + - name: Check proofs using only the extracted source and pinned dependencies + run: | + cd "$PROOFS_EXTRACTED/proofs" + lake --wfail build + lake exe proof-audit --output "$GITHUB_WORKSPACE/build/proofs/sdist-check.json" + lake env leanchecker --fresh Checks > "$GITHUB_WORKSPACE/build/proofs/sdist-replay.log" 2>&1 + python -I -B Checks/audit.py --verify-report "$GITHUB_WORKSPACE/build/proofs/sdist-check.json" + - name: Recheck artifact identities after proof execution + shell: bash + run: | + args=(--verify --sdist build/proofs/dist/*.tar.gz --receipt build/proofs/receipt.json) + for wheel in build/proofs/dist/*.whl; do args+=(--wheel "$wheel"); done + python -I -B dev/proofs/package.py "${args[@]}" + - uses: actions/upload-artifact@v4 + with: + name: proof-distribution-receipt + path: | + build/proofs/receipt.json + build/proofs/sdist-check.json + build/proofs/sdist-replay.log + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: lean-native-wheels + path: build/proofs/dist/*.whl + if-no-files-found: error + + bridge: + name: Lean bridge + needs: [model, distribution] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/download-artifact@v4 + with: + name: lean-model-evidence + path: build/proofs + - uses: actions/download-artifact@v4 + with: + name: lean-native-wheels + path: build/proofs/wheels + - name: Install the selected native wheel + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt numpy + python -m pip install --no-index --no-deps --find-links build/proofs/wheels \ + --report build/proofs/install.json netgraph-core + python - <<'PY' + import json, os + from urllib.parse import unquote, urlparse + data = json.load(open('build/proofs/install.json')) + installed = [x for x in data['install'] if x['metadata']['name'] == 'netgraph-core'] + assert len(installed) == 1 + url = urlparse(installed[0]['download_info']['url']) + assert url.scheme == 'file' + with open(os.environ['GITHUB_ENV'], 'a') as f: + f.write('PROOFS_WHEEL=' + unquote(url.path) + '\n') + PY + - name: Exact native comparisons against current model evidence + run: PROOFS_NATIVE_PYTHON=$(command -v python) bash dev/proofs/run.sh bridge + - uses: actions/upload-artifact@v4 + with: + name: lean-bridge-evidence + path: build/proofs/bridge.json + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 85d5023..8f504bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,8 +48,11 @@ jobs: steps: - uses: actions/checkout@v4 - # Install uv for faster dependency installation (required on macOS, bundled in manylinux) + # Install uv for faster dependency installation (required on macOS, bundled in manylinux). + # An exact version, like the Python tools of .github/ci-constraints.txt. - uses: astral-sh/setup-uv@v4 + with: + version: "0.11.16" # Build wheels using cibuildwheel - name: Build wheels @@ -62,10 +65,24 @@ jobs: # Use uv for faster dependency installation (10-100x faster than pip) CIBW_BUILD_FRONTEND: "build[uv]" + # The pins of .github/ci-constraints.txt for what uv installs without constraints of + # cibuildwheel's own: the build requirements of every wheel (scikit-build-core and its + # dependencies; pybind11 is pinned in pyproject.toml) and the test requirements. uv + # reads UV_CONSTRAINT only when a call passes no -c, so cibuildwheel's own tools keep + # its pins. Linux builds in a container that holds the project at /project. + CIBW_ENVIRONMENT_LINUX: "UV_CONSTRAINT=/project/.github/ci-constraints.txt" + CIBW_ENVIRONMENT_MACOS: UV_CONSTRAINT="$GITHUB_WORKSPACE/.github/ci-constraints.txt" + CIBW_ENVIRONMENT_WINDOWS: UV_CONSTRAINT="$GITHUB_WORKSPACE/.github/ci-constraints.txt" # Target architecture per matrix entry (native compilation, no QEMU) CIBW_ARCHS: "${{ matrix.cibw_archs }}" - CIBW_TEST_COMMAND: 'python -c "import netgraph_core; print(netgraph_core.__version__)"' + # The Python suite on every built wheel (the shipped configuration, hooks off; the + # free-threaded cp314t wheels included): exactly the tests that need a hook skip. + # Windows runs the same guard as dev/run-checks.sh directly (no shell): the wheel lacks + # the private hooks and exactly the hook-only tests skip. + CIBW_TEST_REQUIRES: "pytest==9.1.1" + CIBW_TEST_COMMAND: "PYTHON=python bash {project}/dev/run-checks.sh pytest off -p no:cacheprovider" + CIBW_TEST_COMMAND_WINDOWS: "python {project}/dev/pytest_guard.py off {project}/tests/py -p no:cacheprovider" CIBW_BEFORE_ALL_LINUX: "yum install -y libatomic || (apt-get update && apt-get install -y libatomic1) || apk add libatomic" - uses: actions/upload-artifact@v4 @@ -73,20 +90,59 @@ jobs: name: cibw-wheels-${{ matrix.os }}-${{ matrix.cibw_archs }} path: ./wheelhouse/*.whl + # build fills its isolated environment with a plain pip install, which PIP_CONSTRAINT reaches; + # the same variable pins pip itself and build. build_sdist: name: Build SDist runs-on: ubuntu-latest + env: + PIP_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - name: Build SDist - run: pipx run build --sdist + run: | + python -m pip install -U pip + python -m pip install build + python -m build --sdist - uses: actions/upload-artifact@v4 with: name: cibw-sdist path: dist/*.tar.gz + # An installation built from the source distribution, tested with the Python suite of the same + # commit. pip builds it in an isolated environment of its own, which since pip 26.2 takes only + # build constraints (PIP_BUILD_CONSTRAINT), never PIP_CONSTRAINT: both name the same file. + # PIP_CONSTRAINT pins the venv's pip itself and pytest. + test_sdist: + name: Test an installation built from the SDist + needs: [build_sdist] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/download-artifact@v4 + with: + name: cibw-sdist + path: sdist + - name: Build and install from the SDist + env: + PIP_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt + PIP_BUILD_CONSTRAINT: ${{ github.workspace }}/.github/ci-constraints.txt + run: | + python -m venv "$RUNNER_TEMP/sdist-venv" + "$RUNNER_TEMP/sdist-venv/bin/python" -m pip install -U pip + "$RUNNER_TEMP/sdist-venv/bin/python" -m pip install pytest + "$RUNNER_TEMP/sdist-venv/bin/python" -m pip install -v sdist/*.tar.gz + - name: Python tests against the installation + run: PYTHON="$RUNNER_TEMP/sdist-venv/bin/python" bash dev/run-checks.sh pytest off -p no:cacheprovider + test_wheels_centos_stream_9: name: Test wheels (CentOS Stream 9, py${{ matrix.python-version }}) needs: [build_wheels] @@ -100,6 +156,8 @@ jobs: - name: Install Python run: dnf install -y python${{ matrix.python-version }} python${{ matrix.python-version }}-pip + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: pattern: cibw-wheels-ubuntu-* @@ -108,27 +166,57 @@ jobs: - name: Install and test wheel run: | - python${{ matrix.python-version }} -m pip install --upgrade pip - python${{ matrix.python-version }} -m pip install netgraph-core --find-links wheelhouse/ - python${{ matrix.python-version }} -c "import netgraph_core; print('CentOS Stream 9:', netgraph_core.__version__)" + python${{ matrix.python-version }} -m venv /tmp/wheel-venv + /tmp/wheel-venv/bin/python -m pip install --upgrade -c .github/ci-constraints.txt pip + /tmp/wheel-venv/bin/python -m pip install -c .github/ci-constraints.txt pytest numpy + # --no-index: the wheel built above, never a published one of the same version. + /tmp/wheel-venv/bin/python -m pip install --no-index --find-links wheelhouse/ netgraph-core + PYTHON=/tmp/wheel-venv/bin/python bash dev/run-checks.sh pytest off -p no:cacheprovider + + proofs: + name: Verify release proofs and distributions + needs: [build_sdist, build_wheels] + uses: ./.github/workflows/proofs.yml + with: + sdist-artifact: cibw-sdist + wheels-pattern: cibw-wheels-* publish_pypi: name: Publish to PyPI - needs: [check_version, build_wheels, build_sdist, test_wheels_centos_stream_9] + needs: [check_version, build_wheels, build_sdist, test_sdist, test_wheels_centos_stream_9, proofs] runs-on: ubuntu-latest environment: pypi permissions: + contents: read # Checkout the same tag's artifact-identity verifier. id-token: write # Required for Trusted Publishing (OIDC) # Only publish on tag pushes if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: actions/download-artifact@v4 with: pattern: cibw-* path: dist merge-multiple: true + - uses: actions/download-artifact@v4 + with: + name: proof-distribution-receipt + path: proof-evidence + + - name: Publish exactly the artifacts whose proof distribution was checked + shell: bash + run: | + sdists=(dist/*.tar.gz) + [[ ${#sdists[@]} -eq 1 && -f ${sdists[0]} ]] + args=(--verify --sdist "${sdists[0]}" --receipt proof-evidence/receipt.json) + for wheel in dist/*.whl; do args+=(--wheel "$wheel"); done + python -I -B dev/proofs/package.py "${args[@]}" + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba61abb..c7dd2a6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,10 @@ on: pull_request: workflow_dispatch: +# Every job installs its Python tools with the pins of .github/ci-constraints.txt, pip itself +# first. An editable install builds the package in an isolated environment of pip's own, which +# since pip 26.2 takes only build constraints (--build-constraint), never -c: both name the same +# file. jobs: test: name: Unit tests (${{ matrix.os }}, py${{ matrix.python-version }}) @@ -15,21 +19,37 @@ jobs: matrix: os: [ubuntu-22.04, macos-latest, windows-latest] python-version: ["3.11", "3.12", "3.13", "3.14"] + include: + # Free-threaded CPython (no GIL): the module declares py::mod_gil_not_used(). + - os: ubuntu-22.04 + python-version: "3.14t" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + # bash on every OS: PowerShell would not stop when the first command fails. - name: Install deps + shell: bash run: | - python -m pip install -U pip wheel - python -m pip install -e .[dev] - - name: Run CI checks (lint + C++/Python tests) + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt --build-constraint .github/ci-constraints.txt -e ".[dev]" + - name: Run CI checks (lint + C++ tests with and without the hooks + Python tests) if: runner.os != 'Windows' run: make check-ci + # Windows runs the Python tests through the guard dev/run-checks.sh uses, without the shell + # script: the editable build must carry the private hooks and no test may skip. - name: Run Python tests (Windows) if: runner.os == 'Windows' - run: python -m pytest tests/py/ -v + run: python dev/pytest_guard.py on tests/py -v + # The timing tool of dev/perf compiles against the library (its adapter uses the public + # API and two private functions of the queue rule), and nothing else builds it: its own + # tests, which compile its driver with this checkout's library, run on one job, so a + # library change that breaks the tool fails here. macOS: the platform the tool is made + # and checked on (its build record and memory reading are macOS-first). + - name: Timing tool tests (macOS, py3.14) + if: runner.os == 'macOS' && matrix.python-version == '3.14' + run: make perf-tool-test test-centos-stream-9: name: CentOS Stream 9 (py${{ matrix.python-version }}) @@ -53,12 +73,87 @@ jobs: - name: Install Python deps run: | - python${{ matrix.python-version }} -m pip install -U pip wheel - python${{ matrix.python-version }} -m pip install -e .[dev] + python${{ matrix.python-version }} -m pip install -U -c .github/ci-constraints.txt pip + python${{ matrix.python-version }} -m pip install -c .github/ci-constraints.txt \ + --build-constraint .github/ci-constraints.txt -e ".[dev]" - - name: Run CI checks (lint + C++/Python tests) + - name: Run CI checks (lint + C++ tests with and without the hooks + Python tests) run: PYTHON=python${{ matrix.python-version }} make check-ci + # The C++ tests with MSVC: a Release build with CMake's link-time optimisation (the wheel's + # configuration), with and without the private hooks. + cpp-windows-msvc: + name: C++ tests (windows, MSVC) + runs-on: windows-latest + defaults: + run: + shell: bash + env: + CC: cl + CXX: cl + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build tools + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt cmake ninja + - uses: ilammy/msvc-dev-cmd@v1 + - name: C++ tests, private hooks on + run: bash dev/cpp-tests.sh release + - name: C++ tests, private hooks off + run: bash dev/cpp-tests.sh off + + sanitizers: + name: C++ tests (ubuntu, ASan + UBSan) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build tools + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt cmake ninja + - name: C++ tests under AddressSanitizer and UndefinedBehaviorSanitizer + run: bash dev/cpp-tests.sh sanitize + + thread-sanitizer: + name: C++ tests (macOS, ThreadSanitizer) + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build tools + run: | + python -m pip install -U -c .github/ci-constraints.txt pip + python -m pip install -c .github/ci-constraints.txt cmake ninja + - name: C++ tests under ThreadSanitizer + run: bash dev/cpp-tests.sh tsan + + # The generated type stub python/netgraph_core/__init__.pyi (make stubs-check): no drift from a + # fresh generation out of a wheel of this commit, the stub and py.typed in the wheel and the + # sdist, and the typed clients of tests/typing (valid: no diagnostic; invalid: exactly the + # expected ones) under pinned Pyright against the installed wheel and an installation built + # from the sdist. The script makes its own pinned venvs. Python 3.14: the version the + # committed stub is generated with. + stubs: + name: Type stub and typed clients (ubuntu, py3.14) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + - name: Stub drift, package contents, typed clients (wheel and sdist) + run: PYTHON=python bash dev/stubs/run.sh check + coverage: name: Coverage (ubuntu, py3.11) runs-on: ubuntu-22.04 @@ -67,10 +162,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install deps - run: | - python -m pip install -U pip wheel - python -m pip install -e .[dev] + # make cov builds its own pinned venv (build/coverage-venv) and fails on any failure. - name: Generate coverage (Python + C++) run: make cov - name: Upload coverage artifacts diff --git a/.gitignore b/.gitignore index eca4b33..e0d3023 100644 --- a/.gitignore +++ b/.gitignore @@ -1,229 +1,38 @@ -# Byte-compiled / optimized / DLL files +# Python __pycache__/ *.py[codz] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ *.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -.benchmarks/ -cover/ - -# Consolidated coverage artifacts -build/coverage/ - -# CTest output if created in project root -Testing/ -CTestTestfile.cmake - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock -#poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -#pdm.lock -#pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -#pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv -env/ venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -.vscode/ - -# Ruff stuff: +.venv/ +.pytest_cache/ .ruff_cache/ -# PyPI configuration file -.pypirc +# Build outputs: CMake and scikit-build directories, distributions, compiled extensions +build/ +dist/ +*.so -# Cursor -# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to -# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data -# refer to https://docs.cursor.com/context/ignore-files -.cursorignore -.cursorindexingignore +# Credentials of `make publish` +.pypirc -# OS-specific -.DS_Store +# The backup and reject files `patch` leaves beside the files it patches +*.orig +*.rej -# IDEs +# Editors and OS +.vscode/ .idea/ *.code-workspace +.DS_Store -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ -.pgo-profile/ +# Local output of browser-automation tools (Playwright's MCP server) +.playwright-mcp/ # Personal Superset lifecycle overrides .superset/config.local.json -# Performance harness run logs -dev/perf/**/*.log +# The performance tool's default output directory +/dev/perf/results/ + +# Lean dependencies, compiled declarations and local cache state. +/proofs/.lake/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dfd90b..3f8522a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,14 +4,12 @@ repos: hooks: - id: check-merge-conflict - id: check-yaml - # Recorded measurement artefacts and review patches must stay byte-exact. - id: end-of-file-fixer - exclude: ^dev/(research/|perf/.*/(results|review-[^/]+)/) - id: trailing-whitespace - exclude: ^dev/(research/|perf/.*/(results|review-[^/]+)/) - id: mixed-line-ending - id: debug-statements + # The revision matches the dev pin of Ruff (pyproject.toml). - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.13 hooks: @@ -19,9 +17,15 @@ repos: args: ["--fix"] - id: ruff-format - - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.401 + # Pyright needs the compiled extension _netgraph_core to resolve the package's imports, so it + # runs as make lint runs it: the pinned Pyright of the project's venv (make dev), whose + # interpreter has the extension built, over the whole project (pyproject.toml's settings). + # An isolated hook environment cannot hold the extension. + - repo: local hooks: - id: pyright - additional_dependencies: ["numpy>=1.22"] - # Pyright reads configuration from pyproject.toml + name: pyright + entry: venv/bin/python -m pyright --pythonpath venv/bin/python + language: system + types_or: [python, pyi] + pass_filenames: false diff --git a/AGENTS.md b/AGENTS.md index 917fa5f..f93beea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,13 +3,20 @@ - Setup: `bash .superset/workspace.sh setup`. Each worktree gets its own venv; setup does not replace the Git hooks shared by worktrees. - Superset **Run**: `bash .superset/workspace.sh check` rebuilds the Python - extension, then runs lint, types, C++ and Python tests. Editable Python imports - alone do not rebuild changed C++. -- For C++ changes, also run `make sanitize-test` (ASan/UBSan; errors must fail). + extension, then runs lint, types, the C++ tests with and without the private + hooks, and the Python tests. Editable Python imports alone do not rebuild + changed C++. +- For C++ changes, also run `make sanitize-test` (ASan/UBSan) and + `make tsan-test` (ThreadSanitizer); errors must fail. - For downstream changes, run NetGraph's `dev/check_core_integration.sh` with this worktree's absolute path. Test the chosen build, not the published wheel. Confirm defects with a reproducer. Max-flow tests need feasibility, conservation -and optimality/cut checks with nonuniform costs, masks and cost-factor cases. -Keep bindings and `_docs.py` consistent. Performance claims need correctness -checks and A/B/A on one quiet machine. Local checks do not replace CI. +and optimality/cut checks with nonuniform costs, excluded nodes and edges, and +cost ceilings (AbsoluteCost, CostStretch). +A binding signature or docstring change needs the regenerated type stub +(`bash dev/stubs/run.sh generate`; `make stubs-check` checks drift, package +contents and typed clients). Performance claims need correctness checks and +A/B/A on one quiet machine (`dev/perf/README.md`; `make perf-tool-test` after +changing that tool; CI runs it on one macOS job, as a library change can break +it). Local checks do not replace CI. diff --git a/CHANGELOG.md b/CHANGELOG.md index e2b5c1a..52275f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,94 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.12.0] - 2026-10-27 + +**BREAKING.** The Core is rebuilt and its whole API changes: the graph works on integer units, one +placement engine with four routing models takes the place of the 0.11 policy and algorithm +objects, and three evaluators take the place of the 0.11 max-flow, batch max-flow and sensitivity +calls. No 0.11 name survives as an alias or a warning. [docs/semantics.md](docs/semantics.md) states +the new behaviour exactly. + +### Changed + +- **BREAKING: amounts are integers.** Capacities, offers, deliveries and drops are whole units of a + resolution `q` (a power of two in `[2^-32, 2^32]`) that each graph carries; `to_units` (rounding + half up) and `as_float` convert physical values, and all arithmetic is exact, with no tolerance. + 0.11 used `float64` amounts. +- **BREAKING: costs are at least 1.** Edge costs are integers `>= 1`; 0.11 accepted a cost of 0. +- **BREAKING: edge numbering.** EdgeIds sort the edges by source, cost and destination, ties kept in + input order, and `input_positions()` maps an EdgeId to its input position. 0.11 sorted by cost, + source and destination and carried caller-supplied external ids. +- **BREAKING: zero-capacity edges stay in routing.** Searches and routes use an edge of capacity 0, + a source that reaches a destination only over it still takes part, and an equal split sends it its + share, which is lost: zero capacity alone never excludes an edge. 0.11's equal split left such + edges out. +- **BREAKING: distances** are `int64`, with `UNREACHABLE` for a node not reached; 0.11 returned + `float64` distances with `inf` by default. +- **BREAKING: failure sensitivity** (`evaluate_edge_failures`) reports, with its baseline, a signed + and exact `delivery_reduction` for every candidate edge, zero and negative values included. 0.11 + tested only saturated edges and returned only the reductions above 1/4096. +- **Worker counts.** Without an explicit `workers`, `evaluate_pairs` and `evaluate_edge_failures` + take their worker count from `NGRAPH_CORE_BATCH_THREADS` and `NGRAPH_CORE_SENSITIVITY_THREADS`, + and use 1 when the variable is unset or invalid (0.11: the hardware concurrency); a supplied + `ShortestPathCache` defaults to 1. + +### Added + +- **Graph and units**: `Graph`, `Eligibility` (the nodes and edges a call may use, built from + excluded node and edge ids; a failure is an exclusion), `to_units`, `as_float`. +- **Routes and searches**: `PredecessorDAG` and `Path`; `spf` (forward or reverse, to every node or + to the nearest targets, the structure containing every shortest route or one canonical path), + `ksp` with exact cost ceilings (`AbsoluteCost`, `CostStretch`) and `resolve_to_paths`; + `RouteTieBreak`, the one rule that orders equal-cost choices. +- **Ledger**: `FlowLedger`, which keeps the traffic each edge carries apart from the bandwidth T1 + tunnels reserve on it, and its `FlowRecord` snapshots. +- **Policies**: `FlowPolicy` and `FlowPolicyConfig` route one demand from source nodes to + destination nodes with `LOSSLESS` or `LOSSY` admission. Choose `ShortestPaths` (ECMP, WCMP or a + controller on shortest routes), `DistributedTE` (T1 in the docs: RSVP-TE-style tunnels set up by + each source router), `CentralizedTE` (an ideal controller, optionally with equal-bandwidth + tunnels, T2 in the docs) or `Pinned` (replay of a saved program). +- **Source allocation**: `EQUAL` splits the demand equally between its sources, `OPTIMIZED` lets + the controller choose, and `WEIGHTED` follows declared weights. +- **Policy operations**: `place`, `evaluate`, `replace` and `forward` return a `PlacementResult`; + `remove` takes the policy's traffic out. +- **Forwarding programs**: `ForwardingProgram`, produced by every computing model and built by + `ForwardingProgram.from_routes` and `ForwardingProgram.from_dag`. +- **Several demands at once**: `apply_actions` with `Place`, `Replace` and `Forward` actions, one + after another (`ORDERED`) or at the same time (`SIMULTANEOUS`) with the losses on shared links + divided between the demands. +- **Shortest-path cache**: `ShortestPathCache`, building or frozen, shared by the evaluators' + workers. +- **Evaluators**: `evaluate_demand`, `evaluate_pairs` and `evaluate_edge_failures`, with one + `EvaluationOptions` and an optional finite offer, which every model accepts. +- **Guarantees**: every public call either completes or changes nothing, except that running out of + memory inside pybind11's own object construction may crash the process, and a supplied + shortest-path cache may keep completed entries and lookup statistics; native work releases the + GIL, placements included, and calls on one ledger take effect one at a time. +- **Proofs**: the round-robin rule that splits whole units among members in order, and the + increments it gives when an offer grows over several calls, have machine-checked Lean proofs of + their stated properties, explained with worked examples in + [docs/formal/theorems/round-robin.md](docs/formal/theorems/round-robin.md). Nothing changes at + run time, and installing the package does not need Lean. +- **Typing**: the package ships `py.typed` and a type stub that carries the docstrings. +- **C++**: the same API in self-contained headers under `netgraph/core/` (`types.hpp`, `graph.hpp`, + `eligibility.hpp`, `options.hpp`, `route_structures.hpp`, `forwarding_program.hpp`, `search.hpp`, + `flow_record.hpp`, `flow_ledger.hpp`, `flow_policy.hpp`, `shortest_path_cache.hpp`, + `evaluators.hpp`). + +### Removed + +- **BREAKING:** the 0.11 Python API: `StrictMultiDiGraph`, `FlowGraph`, `FlowState`, `Algorithms`, + `Backend` with its `Graph` handle, `EdgeSelection`, `EdgeTieBreak`, `FlowPlacement`, `PathAlg`, + `PredDAG`, `FlowIndex`, `FlowSummary`, `MinCut`, the 0.11 forms of `FlowPolicy` and + `FlowPolicyConfig`, and the profiling functions `profiling_enabled`, `profiling_dump` and + `profiling_reset`. +- **BREAKING:** the 0.11 C++ headers `algorithms.hpp`, `backend.hpp`, `constants.hpp`, `flow.hpp`, + `flow_graph.hpp`, `flow_state.hpp`, `k_shortest_paths.hpp`, `max_flow.hpp`, `profiling.hpp`, + `shortest_paths.hpp` and `strict_multidigraph.hpp`, and the 0.11 contents of `flow_policy.hpp`, + `options.hpp` and `types.hpp`. +- The environment variables `NGRAPH_CORE_PROFILE` and `NGRAPH_CORE_SPF_QUEUE`. + ## [0.11.0] - 2026-10-02 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f08d4c..528c972 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,47 +1,34 @@ cmake_minimum_required(VERSION 3.23) project(netgraph_core LANGUAGES CXX) -# Silence pybind11 FindPython compatibility warning by selecting the modern mode -set(PYBIND11_FINDPYTHON NEW CACHE STRING "Use modern FindPython in pybind11") -# Optional C++ tests + +# ============================================================================= +# Options. Every build states them on its command line (pyproject.toml for pip builds, +# dev/cpp-tests.sh for the C++ test builds), so a value cached by an earlier configure of the same +# build directory never decides a later build. +# ============================================================================= + option(NETGRAPH_CORE_BUILD_TESTS "Build C++ unit tests" OFF) -if(NETGRAPH_CORE_BUILD_TESTS) - include(FetchContent) - FetchContent_Declare( - googletest - URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - ) - FetchContent_MakeAvailable(googletest) - enable_testing() -add_executable(netgraph_core_tests - tests/cpp/smoke_test.cpp - tests/cpp/flow_policy_tests.cpp - tests/cpp/strict_multidigraph_tests.cpp - tests/cpp/shortest_paths_tests.cpp - tests/cpp/flow_state_tests.cpp - tests/cpp/flow_graph_tests.cpp - tests/cpp/max_flow_tests.cpp - tests/cpp/k_shortest_paths_tests.cpp - tests/cpp/masking_tests.cpp -) - target_link_libraries(netgraph_core_tests PRIVATE netgraph_core GTest::gtest_main) - target_include_directories(netgraph_core_tests PRIVATE tests/cpp) - if(NETGRAPH_CORE_COVERAGE AND (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")) - # Atomic counters: see the NETGRAPH_CORE_COVERAGE block below. - target_compile_options(netgraph_core_tests PRIVATE -O0 -g --coverage) - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - target_compile_options(netgraph_core_tests PRIVATE -fprofile-update=atomic) - endif() - target_link_options(netgraph_core_tests PRIVATE --coverage) - endif() - include(GoogleTest) - gtest_discover_tests(netgraph_core_tests) +# The private hooks (test controls, and diagnostics such as the placement trace that the tests +# compare with the reference under tests/reference/policy/; neither is public API): +# ON for the editable development build (pyproject.toml) and the hooks-on test suites +# (dev/cpp-tests.sh), OFF for a wheel and for the hooks-off test suite. +option(NETGRAPH_CORE_PRIVATE_HOOKS "Build the private test hooks into the library and the extension" OFF) +option(NETGRAPH_CORE_COVERAGE "Enable C++ coverage instrumentation" OFF) +option(NETGRAPH_CORE_SANITIZE "Enable Address/Undefined sanitizers" OFF) +option(NETGRAPH_CORE_TSAN "Enable ThreadSanitizer" OFF) +option(NETGRAPH_CORE_NATIVE "Optimize for current CPU (not portable)" OFF) +if(NETGRAPH_CORE_SANITIZE AND NETGRAPH_CORE_TSAN) + # ThreadSanitizer cannot be combined with AddressSanitizer in one binary. + message(FATAL_ERROR "NETGRAPH_CORE_SANITIZE and NETGRAPH_CORE_TSAN are mutually exclusive") endif() +# Settings for every target: CMake applies them to targets created later, so they come first. set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +# Silence pybind11 FindPython compatibility warning by selecting the modern mode +set(PYBIND11_FINDPYTHON NEW CACHE STRING "Use modern FindPython in pybind11") # macOS: enforce consistent deployment target and architecture for wheels/builds if(APPLE) @@ -59,81 +46,211 @@ if(APPLE) endif() endif() -# Dependencies -find_package(pybind11 3 CONFIG QUIET) +# Dependencies. pybind11 is exactly the version pinned in pyproject.toml's build requirements, +# whose licence notice the wheel ships (third_party/pybind11/LICENSE). +find_package(pybind11 3.1.0 EXACT CONFIG QUIET) if(NOT pybind11_FOUND) - message(STATUS "pybind11 not found via config; using FetchContent") + message(STATUS "pybind11 3.1.0 not found via config; using FetchContent") include(FetchContent) FetchContent_Declare( pybind11 - GIT_REPOSITORY https://github.com/pybind/pybind11.git - GIT_TAG v3.0.0 + URL https://github.com/pybind/pybind11/archive/refs/tags/v3.1.0.tar.gz + URL_HASH SHA256=ef712655692a2e9bf7bb7874c022564a45f91d847ddee987e720cd9e28849665 DOWNLOAD_EXTRACT_TIMESTAMP TRUE ) FetchContent_MakeAvailable(pybind11) endif() +# ============================================================================= +# Targets +# ============================================================================= add_library(netgraph_core STATIC - src/strict_multidigraph.cpp - src/shortest_paths.cpp - src/k_shortest_paths.cpp - src/max_flow.cpp - src/flow_state.cpp - src/flow_graph.cpp + src/concurrency.cpp + src/graph.cpp + src/eligibility.cpp + src/min_cost_flow.cpp + src/route_structures.cpp + src/search.cpp + src/route_kernels.cpp + src/flow_ledger.cpp + src/shortest_path_cache.cpp src/flow_policy.cpp - src/cpu_backend.cpp - src/profiling.cpp + src/evaluators.cpp + src/forwarding_program.cpp + src/forwarding.cpp + src/distributed_te.cpp + src/centralized_te_tunnels.cpp ) target_include_directories(netgraph_core PUBLIC $ $ ) -## C++ standard is set globally above. +# PUBLIC: the library, the extension and the tests see one value (0 or 1). +target_compile_definitions(netgraph_core PUBLIC NETGRAPH_CORE_PRIVATE_HOOKS=$) target_compile_features(netgraph_core PUBLIC cxx_std_20) -# Warnings -if(MSVC) - target_compile_options(netgraph_core PRIVATE /W4 /permissive-) -else() - target_compile_options(netgraph_core PRIVATE -Wall -Wextra -Wpedantic) -endif() - -# Python extension -pybind11_add_module(_netgraph_core bindings/python/module.cpp) +# Python extension: the module entry and its registration files (bindings/python/), in the +# order the entry calls them; the shared helpers are headers there. +pybind11_add_module(_netgraph_core + bindings/python/module.cpp + bindings/python/options.cpp + bindings/python/units.cpp + bindings/python/eligibility.cpp + bindings/python/graph.cpp + bindings/python/flow_solver_hook.cpp + bindings/python/predecessor_dag.cpp + bindings/python/path.cpp + bindings/python/flow_record.cpp + bindings/python/flow_ledger.cpp + bindings/python/forwarding_program.cpp + bindings/python/routing_models.cpp + bindings/python/flow_policy_config.cpp + bindings/python/placement_result.cpp + bindings/python/shortest_path_cache.cpp + bindings/python/flow_policy.cpp + bindings/python/apply_actions.cpp + bindings/python/evaluators.cpp + bindings/python/path_enumeration.cpp + bindings/python/search.cpp + bindings/python/hooks.cpp + bindings/python/concurrency_hooks.cpp +) +# The library's PUBLIC usage requirements (C++20, the hooks flag, the libc++ settings below) reach +# the extension through this link. target_link_libraries(_netgraph_core PRIVATE netgraph_core) -target_compile_features(_netgraph_core PRIVATE cxx_std_20) if(APPLE) # Prefer maximum runtime compatibility for libc++ on macOS target_compile_definitions(netgraph_core PUBLIC _LIBCPP_DISABLE_AVAILABILITY=1 _LIBCPP_ABI_VERSION=1) - target_compile_definitions(_netgraph_core PRIVATE _LIBCPP_DISABLE_AVAILABILITY=1 _LIBCPP_ABI_VERSION=1) endif() +# The targets that get the project's warnings and either the link-time optimisation (where the +# compiler supports it) or, in a coverage or sanitizer build, the instrumentation: the library, +# the extension and, in a test build, the two test executables (added below). The header +# self-containment check below is not one of them. The extra code-generation flags go to the +# library and the extension only. +set(NETGRAPH_CORE_PROJECT_TARGETS netgraph_core _netgraph_core) + +if(NETGRAPH_CORE_BUILD_TESTS) + include(FetchContent) + FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip + URL_HASH SHA256=1f357c27ca988c3f7c6b4bf68a9395005ac6761f034046e9dde0896e3aba00e4 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + ) + # MSVC: link googletest against the same (shared) C runtime as the project. + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + # googletest itself stays at C++17 (under C++20 its char8_t printer warns with Clang 21); the + # project's tests below are C++20 like the library. + set_target_properties(gtest gtest_main gmock gmock_main PROPERTIES CXX_STANDARD 17) + enable_testing() + add_executable(netgraph_core_tests + tests/cpp/graph_tests.cpp + tests/cpp/graph_layout_tests.cpp + tests/cpp/graph_concurrency_tests.cpp + tests/cpp/concurrency_tests.cpp + tests/cpp/min_cost_flow_tests.cpp + tests/cpp/min_cost_flow_deletion_tests.cpp + tests/cpp/eligibility_tests.cpp + tests/cpp/route_structures_tests.cpp + tests/cpp/search_tests.cpp + tests/cpp/ksp_tests.cpp + tests/cpp/shortest_path_cache_tests.cpp + tests/cpp/route_kernels_tests.cpp + tests/cpp/shortest_paths_creation_tests.cpp + tests/cpp/flow_ledger_tests.cpp + tests/cpp/flow_policy_fixture_tests.cpp + tests/cpp/flow_policy_tests.cpp + tests/cpp/epoch_static_tests.cpp + tests/cpp/weighted_source_tests.cpp + tests/cpp/zero_capacity_tests.cpp + tests/cpp/pooled_source_allocation_tests.cpp + tests/cpp/program_tests.cpp + tests/cpp/joint_forwarding_tests.cpp + tests/cpp/apply_actions_tests.cpp + tests/cpp/distributed_te_tests.cpp + tests/cpp/centralized_te_tunnels_tests.cpp + tests/cpp/contract_examples_tests.cpp + tests/cpp/public_names_tests.cpp + ) + target_compile_definitions(netgraph_core_tests PRIVATE NETGRAPH_CORE_TEST_DATA="${CMAKE_CURRENT_SOURCE_DIR}/tests/data") + # Allocation-failure tests replace the global operator new: a separate executable. + add_executable(netgraph_core_fault_tests + tests/cpp/fault_injection.cpp + tests/cpp/flow_ledger_fault_tests.cpp + tests/cpp/flow_policy_fault_tests.cpp + tests/cpp/apply_actions_fault_tests.cpp + tests/cpp/min_cost_flow_deletion_fault_tests.cpp + tests/cpp/search_fault_tests.cpp + tests/cpp/graph_fault_tests.cpp + ) + foreach(tgt netgraph_core_tests netgraph_core_fault_tests) + target_link_libraries(${tgt} PRIVATE netgraph_core GTest::gtest_main) + target_include_directories(${tgt} PRIVATE tests/cpp) + endforeach() + # Every public header compiles on its own: one generated source file per header, holding only + # its include, built with the test executables. No public header names a private detail/ + # header in an #include (the configure stops on one). + file(GLOB NETGRAPH_CORE_PUBLIC_HEADERS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/include + CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/include/netgraph/core/*.hpp) + set(NETGRAPH_CORE_HEADER_CHECKS) + foreach(header ${NETGRAPH_CORE_PUBLIC_HEADERS}) + file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/include/${header} private_includes + REGEX "^[ \t]*#[ \t]*include[ \t]*[\"<][^\">]*detail/") + if(private_includes) + message(FATAL_ERROR "public header ${header} includes a private detail/ header: ${private_includes}") + endif() + string(MAKE_C_IDENTIFIER ${header} id) + file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/header_checks/${id}.cpp CONTENT "#include \"${header}\"\n") + list(APPEND NETGRAPH_CORE_HEADER_CHECKS ${CMAKE_CURRENT_BINARY_DIR}/header_checks/${id}.cpp) + endforeach() + add_library(netgraph_core_header_checks OBJECT ${NETGRAPH_CORE_HEADER_CHECKS}) + target_link_libraries(netgraph_core_header_checks PRIVATE netgraph_core) + add_dependencies(netgraph_core_tests netgraph_core_header_checks) + list(APPEND NETGRAPH_CORE_PROJECT_TARGETS netgraph_core_tests netgraph_core_fault_tests) + include(GoogleTest) + gtest_discover_tests(netgraph_core_tests) + # The fault sweeps fail every allocation of a call in turn, each with a clean retry, and some + # create and join worker threads: under ASan on a CI runner one sweep took nearly ten minutes. + # Every attempt is kept; each fault test gets thirty minutes. + gtest_discover_tests(netgraph_core_fault_tests PROPERTIES TIMEOUT 1800) +endif() + +# Warnings: the project's set on the library, the extension and the test executables (the list +# above); third-party code keeps its own. +if(MSVC) + set(NETGRAPH_CORE_WARNINGS /W4 /permissive-) +else() + set(NETGRAPH_CORE_WARNINGS -Wall -Wextra -Wpedantic) +endif() +foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) + target_compile_options(${tgt} PRIVATE ${NETGRAPH_CORE_WARNINGS}) +endforeach() + # ============================================================================= # Performance optimizations (disabled when coverage/sanitizers are enabled) # ============================================================================= -option(NETGRAPH_CORE_COVERAGE "Enable C++ coverage instrumentation" OFF) -option(NETGRAPH_CORE_SANITIZE "Enable Address/Undefined sanitizers" OFF) -option(NETGRAPH_CORE_NATIVE "Optimize for current CPU (not portable)" OFF) -option(NETGRAPH_CORE_FAST_MATH "Enable fast math (may affect precision)" OFF) -option(NETGRAPH_CORE_PGO_GENERATE "Instrument for PGO profile collection" OFF) -option(NETGRAPH_CORE_PGO_USE "Use PGO profile data (set NETGRAPH_CORE_PGO_DIR)" OFF) -set(NETGRAPH_CORE_PGO_DIR "${CMAKE_BINARY_DIR}/pgo" CACHE PATH "Directory for PGO profile data") - -# Only apply optimizations when not in debug/instrumentation mode -if(NOT NETGRAPH_CORE_COVERAGE AND NOT NETGRAPH_CORE_SANITIZE) - # LTO (Link Time Optimization) - enables cross-module inlining +# Apply the extra optimization settings when coverage and sanitizers are off (whatever the build +# type: a Debug build gets them too). +if(NOT NETGRAPH_CORE_COVERAGE AND NOT NETGRAPH_CORE_SANITIZE AND NOT NETGRAPH_CORE_TSAN) + # LTO (Link Time Optimization) - enables cross-module inlining. LANGUAGES CXX: googletest + # enables C as well, and the check would otherwise require LTO support for C too and turn LTO + # off in every test build. The test executables are LTO-linked like the extension, so the C++ + # tests exercise the code generation that ships. include(CheckIPOSupported) - check_ipo_supported(RESULT ipo_supported OUTPUT ipo_error) + check_ipo_supported(RESULT ipo_supported OUTPUT ipo_error LANGUAGES CXX) if(ipo_supported) message(STATUS "Enabling LTO (Link Time Optimization)") - set_target_properties(netgraph_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) - set_target_properties(_netgraph_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) + set_target_properties(${NETGRAPH_CORE_PROJECT_TARGETS} PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) else() message(STATUS "LTO not supported: ${ipo_error}") endif() + # MSVC gets no extra flags: its build type's optimisation (/O2 in Release) and CMake's LTO (/GL + # with the matching /LTCG link option) suffice. if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") # GCC-only: allow aggressive inlining by promising symbols won't be interposed if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -155,78 +272,33 @@ if(NOT NETGRAPH_CORE_COVERAGE AND NOT NETGRAPH_CORE_SANITIZE) target_compile_options(netgraph_core PRIVATE -march=native) target_compile_options(_netgraph_core PRIVATE -march=native) endif() - - # Fast math - trades IEEE compliance for speed - if(NETGRAPH_CORE_FAST_MATH) - message(STATUS "Enabling fast math optimizations") - target_compile_options(netgraph_core PRIVATE -ffast-math) - target_compile_options(_netgraph_core PRIVATE -ffast-math) - endif() - - # PGO (Profile-Guided Optimization) - uses runtime profile to guide inlining decisions - # GCC: -fprofile-generate/-fprofile-use, Clang: -fprofile-instr-* - if(NETGRAPH_CORE_PGO_GENERATE) - message(STATUS "PGO: Instrumenting for profile generation -> ${NETGRAPH_CORE_PGO_DIR}") - file(MAKE_DIRECTORY ${NETGRAPH_CORE_PGO_DIR}) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - target_compile_options(_netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - target_link_options(netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - target_link_options(_netgraph_core PRIVATE -fprofile-generate=${NETGRAPH_CORE_PGO_DIR}) - else() - # Clang/AppleClang: use LLVM instrumentation PGO - target_compile_options(netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - target_compile_options(_netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - target_link_options(netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - target_link_options(_netgraph_core PRIVATE -fprofile-instr-generate=${NETGRAPH_CORE_PGO_DIR}/default.profraw) - endif() - elseif(NETGRAPH_CORE_PGO_USE) - message(STATUS "PGO: Using profile data from ${NETGRAPH_CORE_PGO_DIR}") - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(netgraph_core PRIVATE -fprofile-use=${NETGRAPH_CORE_PGO_DIR} -fprofile-correction) - target_compile_options(_netgraph_core PRIVATE -fprofile-use=${NETGRAPH_CORE_PGO_DIR} -fprofile-correction) - else() - # Clang/AppleClang: use merged .profdata file - target_compile_options(netgraph_core PRIVATE -fprofile-instr-use=${NETGRAPH_CORE_PGO_DIR}/default.profdata) - target_compile_options(_netgraph_core PRIVATE -fprofile-instr-use=${NETGRAPH_CORE_PGO_DIR}/default.profdata) - endif() - endif() - elseif(MSVC) - # /O2: full optimization, /GL: whole program optimization (MSVC's LTO) - target_compile_options(netgraph_core PRIVATE /O2 /GL) - target_compile_options(_netgraph_core PRIVATE /O2 /GL) - if(NETGRAPH_CORE_FAST_MATH) - target_compile_options(netgraph_core PRIVATE /fp:fast) - target_compile_options(_netgraph_core PRIVATE /fp:fast) - endif() endif() endif() # ============================================================================= -# Debug/instrumentation options (mutually exclusive with optimizations above) +# Coverage and sanitizer options (mutually exclusive with the extra optimizations above). A request +# the compiler cannot honour stops the configure: a build that ran its tests without the requested +# checker would otherwise report success. # ============================================================================= if(NETGRAPH_CORE_COVERAGE) message(STATUS "Enabling coverage instrumentation (disables optimizations)") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - # batch_max_flow and sensitivity_analysis run worker threads, and the default - # non-atomic profile counters corrupt under concurrent updates (gcov then - # emits impossible values such as "branch 4 taken -1"). - set(COV_FLAGS -O0 -g --coverage) - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - list(APPEND COV_FLAGS -fprofile-update=atomic) - endif() - foreach(tgt netgraph_core _netgraph_core) + # Atomic counters (GCC and Clang): the evaluators' worker threads corrupt the default + # non-atomic profile counters under concurrent updates (gcov then emits impossible values + # such as "branch 4 taken -1"). + set(COV_FLAGS -O0 -g --coverage -fprofile-update=atomic) + foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) target_compile_options(${tgt} PRIVATE ${COV_FLAGS}) target_link_options(${tgt} PRIVATE --coverage) endforeach() else() - message(WARNING "NETGRAPH_CORE_COVERAGE requires GCC or Clang") + message(FATAL_ERROR "NETGRAPH_CORE_COVERAGE requires GCC or Clang") endif() endif() if(NETGRAPH_CORE_SANITIZE) - message(STATUS "Enabling sanitizers (disables optimizations)") + message(STATUS "Enabling sanitizers (without the extra optimizations)") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") # Must be a CMake list: a single quoted string reaches the compiler as one # argument and clang rejects "-fsanitize=address,undefined -fno-omit-frame-pointer". @@ -237,18 +309,34 @@ if(NETGRAPH_CORE_SANITIZE) # and then silently ignored, which is indistinguishable from not checking at all. set(SAN_FLAGS -fsanitize=address,undefined -fno-omit-frame-pointer -fno-sanitize-recover=undefined) - set(SAN_TARGETS netgraph_core _netgraph_core) - if(TARGET netgraph_core_tests) - # The test executable links the sanitized static library, so it must be - # built and linked with the same flags to pull in the sanitizer runtime. - list(APPEND SAN_TARGETS netgraph_core_tests) - endif() - foreach(tgt ${SAN_TARGETS}) + # The test executables link the sanitized static library, so they are built and linked with + # the same flags to pull in the sanitizer runtime. + foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) target_compile_options(${tgt} PRIVATE ${SAN_FLAGS}) target_link_options(${tgt} PRIVATE ${SAN_FLAGS}) endforeach() else() - message(WARNING "NETGRAPH_CORE_SANITIZE requires GCC or Clang") + message(FATAL_ERROR "NETGRAPH_CORE_SANITIZE requires GCC or Clang") + endif() +endif() + +if(NETGRAPH_CORE_TSAN) + message(STATUS "Enabling ThreadSanitizer (without the extra optimizations)") + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + # The data-race check of the evaluators' worker threads and the shared ShortestPathCache. A CMake + # list, as SAN_FLAGS above. A report makes the process exit non-zero (TSan's exit code 66, + # or an abort under the halt_on_error=1 of dev/cpp-tests.sh), so ctest fails on any race. -O1 + # as the TSan documentation advises (the Debug build type adds -g); the instrumented + # suites at -O0 would be several times slower for no extra coverage. + set(TSAN_FLAGS -fsanitize=thread -fno-omit-frame-pointer -O1) + # The test executables link the instrumented static library: same compile and link flags, so + # they carry the TSan runtime and their own accesses are instrumented too. + foreach(tgt ${NETGRAPH_CORE_PROJECT_TARGETS}) + target_compile_options(${tgt} PRIVATE ${TSAN_FLAGS}) + target_link_options(${tgt} PRIVATE ${TSAN_FLAGS}) + endforeach() + else() + message(FATAL_ERROR "NETGRAPH_CORE_TSAN requires GCC or Clang") endif() endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6cd3577..34d629e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ NetGraph-Core is a hybrid C++/Python project. You will need: - Python 3.11+ - C++20 compatible compiler (GCC 10+, Clang 12+, MSVC 2019+) - CMake 3.23+ -- Ninja (recommended) +- Ninja (the C++ test runner uses it; `make dev` installs CMake and Ninja into the venv) ### 1. Clone the repository @@ -37,7 +37,11 @@ This will: ### Running Tests -Run all checks (linting + C++ tests + Python tests): +Run all checks: rebuild the editable extension, then lint (ruff, pyright), the check that the +policy fixture data still equals its frozen reference (`tests/reference/policy/README.md`), the +C++ tests with and without the private test hooks, and the Python tests. It only verifies and +never edits a file; `make check-ci` is the same without the rebuild (CI installs the package +itself): ```bash make check @@ -46,8 +50,12 @@ make check Run specific test suites: ```bash -make cpp-test # C++ tests (GoogleTest) -make test # Python tests (pytest) +make cpp-test # C++ tests (GoogleTest), private hooks on +make cpp-test-off # C++ tests without the hooks (the shipped configuration) +make test # Python tests (pytest) +make sanitize-test # C++ tests under ASan/UBSan (required for C++ changes) +make tsan-test # C++ tests under ThreadSanitizer (required for C++ changes) +make cov # Coverage of the C++ and Python suites (its own venv and builds) ``` ### Code Style @@ -55,26 +63,54 @@ make test # Python tests (pytest) - **Python**: We use `ruff` for linting and formatting, and `pyright` for static type checking. - **C++**: We follow standard C++20 practices. -Auto-fix Python formatting: +Apply fixes (these edit files): ```bash -make format +make format # ruff formatting +make hooks # every pre-commit hook, including ruff --fix and whitespace fixers ``` +The commit hook and `make hooks` run Pyright as `make lint` does, from the `make dev` venv: it +needs the built extension to resolve the package's imports. + +### Type Stub + +`python/netgraph_core/__init__.pyi` is generated from the compiled module. After changing a +binding's signature or docstring, run `bash dev/stubs/run.sh generate` and commit the stub; +`make stubs-check` (also in CI) fails when the stub drifts or the typed client checks fail. + +### Behaviour Documentation + +[docs/semantics.md](docs/semantics.md) is the maintained statement of what the library computes. +A change to observable behaviour (a result, an order, an outcome, an error, a default) updates it +in the same change, and its examples must still print what the document shows: +`tests/py/test_docs_examples.py` runs them, and README's quick start, in the Python tests. + +### Mathematical Proofs + +`proofs/` is a Lean project that proves properties of the round-robin rule; +[proofs/README.md](proofs/README.md) gives the commands and +[dev/proofs/README.md](dev/proofs/README.md) the rules for adding or changing a theorem. Ordinary +native and Python development does not need Lean. A theorem is accepted only with a plain +explanation of its statement and its proof, supporting lemmas included, that an engineer without +mathematical training can follow: a worked example and a boundary example, and a failing case +for each assumption that can fail, with every input, step and result explained. + ## Release Process Releases are automated via GitHub Actions when a new tag is pushed. 1. Move the pending notes into a new `## [X.Y.Z] - YYYY-MM-DD` section in `CHANGELOG.md`, dated the day you tag. Mark anything that breaks consumers as **BREAKING**. -2. Bump `version` in `pyproject.toml` to the same `X.Y.Z`. CI does not cross-check the - tag against this value, so a mismatch publishes the wrong version silently. -3. Commit and push. +2. Bump `version` in `pyproject.toml` to the same `X.Y.Z`. Release CI compares the tag + with this value and refuses to publish on a mismatch. +3. Commit and push, then wait for the `Lean verification` workflow to pass on the pushed + branch: publication waits for the same workflow on the tag, so a failure there blocks it. 4. Create and push a matching tag: ```bash - git tag v0.8.0 - git push origin v0.8.0 + git tag vX.Y.Z + git push origin vX.Y.Z ``` 5. The CI pipeline builds wheels and the sdist, tests them, and publishes to PyPI via diff --git a/Makefile b/Makefile index 3828d0f..feb6218 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ # NetGraph-Core Development Makefile -.PHONY: help venv clean-venv dev install install-native install-pgo check check-ci lint format test qt build clean check-dist publish-test publish info hooks check-python cpp-test cov sanitize-test rebuild +.PHONY: help venv clean-venv dev install install-native check check-ci lint format test build clean check-dist publish-test publish info hooks check-python cpp-test cpp-test-off cov sanitize-test tsan-test stubs-check perf-tool-test rebuild .DEFAULT_GOAL := help # -------------------------------------------------------------------------- # Python interpreter detection # -------------------------------------------------------------------------- -# VENV_BIN: path to local virtualenv bin directory -VENV_BIN := $(PWD)/venv/bin +# VENV_BIN: the checkout's virtualenv bin directory (CURDIR is the directory make runs in, -C included) +VENV_BIN := $(CURDIR)/venv/bin # PY_BEST: scan for newest supported Python (used when creating new venvs) # Supports 3.11-3.14 to match requires-python >=3.11 @@ -41,22 +41,27 @@ help: @echo "Setup & Installation:" @echo " make venv - Create a local virtualenv (./venv)" @echo " make dev - Full development environment (package + dev deps + hooks)" - @echo " make install - Install package (default optimizations: LTO, loop unrolling)" + @echo " make install - Install package (editable, private hooks; LTO, loop unrolling)" @echo " make install-native - Install with CPU-specific optimizations (faster, not portable)" - @echo " make install-pgo - Profile-guided optimization (two-phase build, experimental)" @echo " make clean-venv - Remove virtual environment" @echo " make rebuild - Clean and rebuild (respects CMAKE_ARGS)" @echo "" @echo "Code Quality & Testing:" - @echo " make check - Run pre-commit (auto-fix) + C++/Python tests, then lint" - @echo " make check-ci - Run non-mutating lint + tests (CI entrypoint)" + @echo " make check - Rebuild the editable extension, then everything check-ci runs" + @echo " make check-ci - Lint, fixture data check, the pytest guard's rules, C++ tests (hooks on and off), Python tests" + @echo " (the CI entrypoint)" + @echo " (check and check-ci only verify: they never edit a file)" @echo " make lint - Run only linting (non-mutating: ruff + pyright)" - @echo " make format - Auto-format code with ruff" + @echo " make format - Auto-format code with ruff (edits files)" @echo " make test - Run Python tests (pytest)" - @echo " make qt - Run quick tests only (exclude slow/benchmark)" - @echo " make cpp-test - Build and run C++ tests" - @echo " make cov - Coverage summary + XML + single-page combined HTML" + @echo " make cpp-test - Build and run C++ tests (private hooks on)" + @echo " make cpp-test-off - Build and run C++ tests without the hooks (the shipped configuration)" + @echo " make cov - Coverage (own venv and builds) + XML + single-page combined HTML" @echo " make sanitize-test - Build and run C++ tests with sanitizers" + @echo " make tsan-test - Build and run C++ tests with ThreadSanitizer" + @echo " make stubs-check - Type stub: drift, wheel/sdist contents, typed clients (own venvs)" + @echo " (regenerate the stub: bash dev/stubs/run.sh generate)" + @echo " make perf-tool-test - Tests of the A/B/A timing tool in dev/perf (after changing it; CI: macOS)" @echo "" @echo "Build & Package:" @echo " make build - Build distribution packages" @@ -67,7 +72,7 @@ help: @echo "" @echo "Utilities:" @echo " make info - Show project information" - @echo " make hooks - Run pre-commit on all files" + @echo " make hooks - Run pre-commit on all files (applies its fixes)" @echo " make check-python - Check if venv Python matches best available" # Allow callers to pass CMAKE_ARGS and MACOSX_DEPLOYMENT_TARGET consistently @@ -120,48 +125,34 @@ install: @echo "📦 Installing package (editable, default optimizations)" @$(DEV_ENV) $(PIP) install -e . +# The project options are stated in pyproject.toml for every build; a variant overrides one of them +# with -C (a config setting), so it cannot stay in the shared build directory's cache. install-native: @echo "📦 Installing with native CPU optimizations (-march=native)" @echo " Note: Binary will only work on this CPU architecture" - @$(ENV_MACOS) $(ENV_CC) $(ENV_CXX) CMAKE_ARGS="$(strip $(CMAKE_ARGS) -DNETGRAPH_CORE_NATIVE=ON)" $(PIP) install -e . - -# PGO profile stored outside build/ so it survives rebuild -PGO_DIR := $(PWD)/.pgo-profile - -install-pgo: - @echo "📦 PGO Build Phase 1/3: Instrumenting..." - @rm -rf build/ $(PGO_DIR) - @mkdir -p $(PGO_DIR) - @$(ENV_MACOS) $(ENV_CC) $(ENV_CXX) CMAKE_ARGS="$(strip $(CMAKE_ARGS) -DNETGRAPH_CORE_PGO_GENERATE=ON -DNETGRAPH_CORE_PGO_DIR=$(PGO_DIR) -DNETGRAPH_CORE_NATIVE=ON)" $(PIP) install -e . - @echo "📦 PGO Build Phase 2/3: Collecting profile (running benchmark)..." - @$(PYTHON) dev/benchmark_profiling_overhead.py --mesh-size 25 --spf-iters 1000 --flow-iters 200 >/dev/null 2>&1 - @# Clang: merge raw profiles into .profdata (profile may be in cwd or PGO_DIR) - @if ls $(PGO_DIR)/*.profraw >/dev/null 2>&1; then \ - echo " Merging Clang profile data..."; \ - xcrun llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw 2>/dev/null || \ - llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw; \ - elif ls *.profraw >/dev/null 2>&1; then \ - mv *.profraw $(PGO_DIR)/; \ - xcrun llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw 2>/dev/null || \ - llvm-profdata merge -output=$(PGO_DIR)/default.profdata $(PGO_DIR)/*.profraw; \ - fi - @echo "📦 PGO Build Phase 3/3: Rebuilding with profile data..." - @rm -rf build/ - @$(ENV_MACOS) $(ENV_CC) $(ENV_CXX) CMAKE_ARGS="$(strip $(CMAKE_ARGS) -DNETGRAPH_CORE_PGO_USE=ON -DNETGRAPH_CORE_PGO_DIR=$(PGO_DIR) -DNETGRAPH_CORE_NATIVE=ON)" $(PIP) install -e . - @echo "✅ PGO build complete" + @$(DEV_ENV) $(PIP) install -e . -C cmake.define.NETGRAPH_CORE_NATIVE=ON +# Verification only: neither target edits a file (make format and make hooks apply fixes). check: - @PYTHON=$(PYTHON) bash dev/run-checks.sh + @$(MAKE) install + @$(MAKE) check-ci +# The fixture data check: tests/data/policy_fixtures.json still equals what its frozen reference +# computes (tests/reference/policy/README.md; standard library only; -B writes no __pycache__). check-ci: @$(MAKE) lint + @$(PYTHON) -I -B tests/reference/policy/export_policy_fixtures.py --check + @$(PYTHON) -I -B dev/test_pytest_guard.py @$(MAKE) cpp-test - @$(MAKE) test + @$(MAKE) cpp-test-off + @PYTHON=$(PYTHON) bash dev/run-checks.sh pytest on +# Pyright resolves imports (NumPy in the type stub, the extension) through the interpreter it +# is given; without --pythonpath it would take whichever python3 comes first on PATH. lint: @$(RUFF) format --check . @$(RUFF) check . - @$(PYTHON) -m pyright + @$(PYTHON) -m pyright --pythonpath "$$($(PYTHON) -c 'import sys; print(sys.executable)')" format: @$(RUFF) format . @@ -169,9 +160,6 @@ format: test: @$(PYTEST) -qt: - @$(PYTEST) --no-cov -m "not slow and not benchmark" - build: @echo "🏗️ Building distribution packages..." @if $(PYTHON) -c "import build" >/dev/null 2>&1; then \ @@ -211,8 +199,7 @@ publish: clean: @echo "🧹 Cleaning build artifacts and cache files..." @rm -rf build/ dist/ *.egg-info/ || true - @rm -rf .pytest_cache .ruff_cache .mypy_cache htmlcov htmlcov-python .coverage coverage.xml coverage-*.xml .benchmarks .pytest-benchmark || true - @rm -rf Testing CTestTestfile.cmake || true + @rm -rf .pytest_cache .ruff_cache || true @find . -path "./venv" -prune -o -type f -name "*.pyc" -delete 2>/dev/null || true @find . -path "./venv" -prune -o -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true @find . -path "./venv" -prune -o -type f -name "*.pyo" -delete 2>/dev/null || true @@ -246,38 +233,26 @@ hooks: cpp-test: @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh release +cpp-test-off: + @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh off + +# The summary script's own rules first (its refusals make a failed measurement fail; no build). cov: - @echo "📦 Reinstalling with C++ coverage instrumentation..." - @$(PIP) install -U scikit-build-core "pybind11>=2.11" - @PIP_NO_BUILD_ISOLATION=1 CMAKE_ARGS="-DNETGRAPH_CORE_COVERAGE=ON" $(PIP) install -e .'[dev]' - @echo "🧪 Running Python tests with coverage..." - @mkdir -p build/coverage - @$(PYTEST) --cov=netgraph_core --cov-report=term-missing --cov-report=xml:build/coverage/coverage-python.xml - @echo "🛠️ Building and running C++ tests with coverage..." - @BUILD_DIR="build/cpp-tests-cov"; \ - mkdir -p "$$BUILD_DIR"; \ - GEN_ARGS=""; \ - if command -v ninja >/dev/null 2>&1; then GEN_ARGS="-G Ninja"; fi; \ - cmake -S . -B "$$BUILD_DIR" -DNETGRAPH_CORE_BUILD_TESTS=ON -DNETGRAPH_CORE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug $$GEN_ARGS; \ - cmake --build "$$BUILD_DIR" --config Debug -j; \ - ctest --test-dir "$$BUILD_DIR" --output-on-failure || echo "⚠️ Some C++ tests failed (continuing for coverage)" - @echo "📈 Generating C++ coverage XML (gcovr)..." - @$(PYTHON) -m gcovr --root . \ - --object-directory build \ - --object-directory build/cpp-tests-cov \ - --filter 'include/netgraph' --filter 'src' --exclude 'tests' --exclude 'bindings/.*' --exclude '.*pybind11.*' --exclude '_deps/pybind11-src/.*' \ - --gcov-ignore-errors=all \ - --gcov-ignore-parse-errors=negative_hits.warn_once_per_file \ - --xml-pretty -o build/coverage/coverage-cpp.xml - @echo "" - @echo "================ Python + C++ coverage (summary) ================" - @$(PYTHON) dev/coverage_summary.py build/coverage/coverage-python.xml build/coverage/coverage-cpp.xml --html=build/coverage/coverage-combined.html - @echo "" - @echo "✅ Coverage ready in build/coverage/: coverage-python.xml, coverage-cpp.xml, coverage-combined.html" + @$(PYTHON) -I -B dev/test_coverage_summary.py + @PYTHON=$(PYTHON) bash dev/run-checks.sh cov sanitize-test: @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh sanitize +tsan-test: + @PATH="$(VENV_BIN):$$PATH" bash dev/cpp-tests.sh tsan + +stubs-check: + @PYTHON=$(PYTHON) bash dev/stubs/run.sh check + +perf-tool-test: + @$(PYTHON) -B dev/perf/test_tool.py + # Clean + reinstall in dev mode (respects CMAKE_ARGS and MACOSX_DEPLOYMENT_TARGET) # Uses active PYTHON (venv or PATH) to avoid environment mismatches rebuild: clean diff --git a/README.md b/README.md index 6bd2eff..363d4ab 100644 --- a/README.md +++ b/README.md @@ -1,142 +1,122 @@ # NetGraph-Core -C++ graph engine for network flow analysis, traffic engineering simulation, and capacity planning. - -## Overview - -NetGraph-Core provides a specialized graph implementation for networking problems. Key design priorities: - -- **Determinism**: Guaranteed reproducible edge ordering by (cost, src, dst). -- **Flow Modeling**: Native support for multi-commodity flow state, residual tracking, and ECMP/WCMP placement. -- **Performance**: Immutable CSR (Compressed Sparse Row) adjacency, and zero-copy NumPy views over mutable flow state. - -## Core Features - -### 1. Graph Representations - -- **`StrictMultiDiGraph`**: Immutable directed multigraph using CSR adjacency. Supports parallel edges (multi-graph), essential for network topologies. -- **`FlowGraph`**: Topology overlay managing mutable flow state, per-flow edge allocations, and residual capacities. - -### 2. Network Algorithms - -- **Shortest Paths (SPF)**: - - Modified Dijkstra returns a **Predecessor DAG** to capture all equal-cost paths. - - Supports **ECMP** (Equal-Cost Multi-Path) routing. - - Features **node/edge masking** and **residual-aware tie-breaking**. - -- **K-Shortest Paths (KSP)**: - - Yen's algorithm returning DAG-wrapped paths. - - Configurable constraints on cost factors (e.g., paths within 1.5x of optimal). - -- **Explicit Paths**: - - `PredDAG.from_edges(graph, edges)` builds a path bundle from an operator-supplied - edge sequence, usable anywhere a `PredDAG` is accepted. - -- **Max-Flow**: - - **Algorithm**: Iterative augmentation using Successive Shortest Path on residual graphs, pushing flow across full ECMP/WCMP DAGs at each step. For `Proportional` placement with `require_capacity=True`, a final residual completion phase augments with reverse arcs so the result is a true maximum flow; the other modes are placement models and may report less. - - **Traffic Engineering (TE) Mode**: Routing adapts to residual capacity (progressive fill). - - **IP Routing Mode**: Cost-only routing (ECMP/WCMP) ignoring capacity constraints. - -- **Analysis**: - - **Sensitivity Analysis**: Identifies critical edges by removing each saturated edge and measuring how much total flow is *lost*. Supports `shortest_path` mode to analyze only edges used under ECMP routing (IP/IGP networks) vs. full max-flow (SDN/TE networks). - - **Min-Cut**: Computes minimum cuts on residual graphs. - -### 3. Flow Policy Engine - -Unified configuration object (`FlowPolicy`) that models diverse routing behaviors: - -- **Modeling**: Unified configuration for **IP Routing** (static costs) and **Traffic Engineering** (dynamic residuals). -- **Placement Strategies**: - - `EqualBalanced`: **ECMP** (equal splitting) - equal distribution across next-hops and parallel edges. - - `Proportional`: **WCMP** (weighted splitting) - distribution proportional to residual capacity. -- **Lifecycle Management**: Handles demand placement, re-optimization of existing flows, and constraints (path cost, stretch factor, flow counts). -- **Static (Pinned) Paths**: `FlowPolicy.set_static_paths()` pins a demand to explicit path bundles (MPLS-style). One flow per usable bundle; a bundle with no path surviving the failure masks is *down* and carries nothing, since a pinned path does not reroute. - -### 4. Python Integration - -- **Zero-Copy**: `FlowState` and `FlowGraph` `*_view()` methods expose C++ buffers as read-only NumPy arrays that track mutation in place. `StrictMultiDiGraph.*_view()` returns copies instead, so the graph's immutability cannot be violated. -- **Concurrency**: Releases the Python GIL during the long-running algorithms (SPF, KSP, max-flow, placement) to enable threading. `PredDAG.resolve_to_paths` holds the GIL. `batch_max_flow` and `sensitivity_analysis` also use internal worker threads, sized by `NGRAPH_CORE_BATCH_THREADS` / `NGRAPH_CORE_SENSITIVITY_THREADS`. +NetGraph-Core is a C++ engine with Python bindings that computes how traffic crosses a network: +the maximum flow between two sets of nodes, how ECMP or WCMP shortest-path routing, RSVP-TE +tunnels or an ideal controller would carry a demand, and how much a link failure costs. All +amounts are exact integers, so a result never depends on floating-point tolerance. It is a +library for developers who build network analysis tools. For a ready tool, with networks and +failure scenarios in YAML and reports, use [NetGraph](https://github.com/networmix/NetGraph), +which is built on this engine. + +## The main ideas + +- **Units.** Capacities and traffic are whole numbers of units. Each graph carries its + resolution `q`, the size of one unit (a power of two, for example `2**-12` Gbit/s); + `to_units` converts a physical value (rounding half up) and `as_float` converts back. +- **The graph** (`Graph`) is directed and may have parallel links. Each edge has a capacity and + an integer cost of at least 1. Edges are numbered (their `EdgeId`) by sorting on source, cost + and destination. An edge of capacity 0 is an empty link, not a missing one: routing still uses + it, and traffic sent there is lost. +- **A failure** is an `Eligibility`: the nodes and edges a call may not use. +- **A demand** is traffic from a set of sources to a set of destinations. A `FlowPolicy` routes + one demand on a `FlowLedger`, the record of what each edge carries; several demands share one + ledger, and `apply_actions` places them one after another or at the same time. +- **The routing model** decides the routes: + - `ShortestPaths`: routers forwarding on shortest paths, split equally (ECMP) or by + weights (WCMP), or a controller restricted to those paths; + - `DistributedTE` (called T1 in the docs): each source router sets up its own N tunnels of + fixed bandwidth on the shortest path with enough unreserved bandwidth, as RSVP-TE head ends + do; + - `CentralizedTE`: an ideal controller, the cheapest flow that delivers the most; with a + tunnel count (T2), it lays out the same number of single-path tunnels for every source + router, all of one planned bandwidth; + - `Pinned`: forwards a given program, to replay a result or study a failure. +- **Admission** says what happens to traffic that does not fit: `LOSSLESS` admits only what goes + through without loss, `LOSSY` sends everything and reports where it is lost. +- **A program** (`ForwardingProgram`) is what a model decided: the routes and the integer + weights that split traffic over them. `place`, `evaluate`, `replace` and `forward` return a + `PlacementResult`: what was delivered and lost, and where. +- **The evaluators** answer one question each on a fresh ledger at full capacity: + `evaluate_demand` for one demand (with `CentralizedTE` and the optimized source allocation, + the exact maximum flow, its cost curve and its minimum cut), `evaluate_pairs` for many + independent pairs in parallel, and `evaluate_edge_failures` for how much delivery each edge's + removal takes away. +- **Calls are all or nothing.** A failing call raises one exception and leaves every ledger, + policy and cache as it was, with two limits: running out of memory inside pybind11's own + object construction may crash the process, and a supplied shortest-path cache may keep + completed entries and lookup statistics. Native work releases Python's interpreter lock (the + GIL); calls on one ledger take effect one at a time, and different ledgers run in parallel. ## Quick Start +Two paths from node 0 to node 3: 10 Gbit/s at cost 1 per hop, and 5 Gbit/s at cost 2 per hop. +One unit is 1 Gbit/s (`q = 1.0`), so every printed amount is in Gbit/s. + ```python -import numpy as np import netgraph_core as ngc -# Two parallel paths from node 0 to node 3. -graph = ngc.StrictMultiDiGraph.from_arrays( - num_nodes=4, - src=np.array([0, 1, 0, 2], dtype=np.int32), - dst=np.array([1, 3, 2, 3], dtype=np.int32), - capacity=np.array([10.0, 10.0, 5.0, 5.0], dtype=np.float64), - cost=np.array([1, 1, 1, 1], dtype=np.int64), - ext_edge_ids=np.arange(4, dtype=np.int64), # your own stable edge ids -) - -algs = ngc.Algorithms(ngc.Backend.cpu()) -handle = algs.build_graph(graph) - -total, summary = algs.max_flow(handle, 0, 3) -print(total) # 15.0 -print(summary.min_cut.edges.tolist()) # [0, 1] -- bottleneck edge ids +q = 1.0 +graph = ngc.Graph(4, [0, 1, 0, 2], [1, 3, 2, 3], [10, 10, 5, 5], [1, 1, 2, 2], q) + +# The maximum flow: 10 on the cost-2 path, then 5 on the cost-4 path; the minimum cut is the first +# link of each path (edges 0 and 1). +te = ngc.FlowPolicyConfig(routing=ngc.CentralizedTE(source_allocation=ngc.SourceAllocation.OPTIMIZED)) +r = ngc.evaluate_demand(graph, 0, 3, ngc.EvaluationOptions(te, outputs=ngc.OutputOptions(min_cut=True))) +print(r.call_delivered, r.marginal_cost_curve, r.min_cut) # 15 [(2, 10), (4, 5)] [0, 1] + +# ECMP sends 12 on the only shortest path; its first link (edge 0) carries 10 and drops 2. +ecmp = ngc.FlowPolicyConfig(routing=ngc.ShortestPaths(), admission=ngc.Admission.LOSSY) +r = ngc.FlowPolicy(ngc.FlowLedger(graph), 0, 3, ecmp).place(12) +print(r.call_delivered, r.call_edge_drops) # 10 [(0, 2)] + +# T1: two tunnels of 6 for a demand of 12; the first takes the 10 path, the second finds no path +# with 6 unreserved. A reservation does not cap traffic: all 12 go into the tunnel that is up, +# and its path carries 10. +t1 = ngc.FlowPolicyConfig(routing=ngc.DistributedTE(ngc.TunnelCount(2)), admission=ngc.Admission.LOSSY) +r = ngc.FlowPolicy(ngc.FlowLedger(graph), 0, 3, t1).replace(12, design_demand=12) +print([t.status.name for t in r.program.tunnels], r.call_delivered) # ['UP', 'INSUFFICIENT_BANDWIDTH'] 10 ``` +A `marginal_cost_curve` lists `(cost, units)` pairs: the first 10 units cost 2 each, the next 5 +cost 4 each. With a finer resolution the same calls count fractions exactly: +`ngc.to_units(2.5, 2**-12)` is 10240 units. + ## Installation ```bash pip install netgraph-core ``` -Or from source: - -```bash -pip install -e . -``` - -### Build Optimizations - -Default builds include LTO and loop unrolling. For local development: +From source, `pip install .` builds the shipped configuration; `pip install -e .` is the editable +development build, which also carries private test hooks. The package is typed: it ships +`py.typed` and a type stub generated from the compiled module. -```bash -make install-native # CPU-specific optimizations (not portable) -``` +## Documentation -## Repository Structure - -``` -src/ # C++ implementation -include/netgraph/core/ # Public C++ headers -bindings/python/ # pybind11 bindings -python/netgraph_core/ # Python package -tests/cpp/ # C++ tests (googletest) -tests/py/ # Python tests (pytest) -dev/ # Developer scripts (not shipped): checks, coverage, -dev/perf/ # performance tooling and recorded A/B/A results, -dev/research/ # research prototypes and their evidence -``` +- [docs/semantics.md](https://github.com/networmix/NetGraph-Core/blob/main/docs/semantics.md): the exact rules (units and rounding, edge numbering, + which route a search returns, how each model splits traffic, results, failures, the + shortest-path cache and the evaluators), with worked examples that the tests run. +- The type stub `netgraph_core/__init__.pyi`: every signature, with its docstring. +- The C++ API: the headers under [include/netgraph/core/](https://github.com/networmix/NetGraph-Core/tree/main/include/netgraph/core); their names + are listed in the last section of the semantics. +- [proofs/](https://github.com/networmix/NetGraph-Core/blob/main/proofs/README.md): machine-checked Lean proofs of the round-robin splitting rule, + explained in [docs/formal/theorems/](https://github.com/networmix/NetGraph-Core/blob/main/docs/formal/theorems/round-robin.md). They prove the + mathematical rule, not the C++ code; nothing in the build or the package needs Lean. ## Development -```bash -make dev # Setup: venv, dependencies, pre-commit hooks -make check # Run all tests and linting (auto-fix formatting) -make check-ci # Strict checks without auto-fix (for CI) -make test # Python tests -make cpp-test # C++ tests only -make cov # Combined coverage report (C++ + Python) -``` - -Performance work (harnesses, old/new/old protocol, recorded measurements) is -described in [dev/perf/README.md](dev/perf/README.md). +[CONTRIBUTING.md](https://github.com/networmix/NetGraph-Core/blob/main/CONTRIBUTING.md) covers setup, the test targets and the release process. In +short: `make dev` sets up a venv with the editable build and the pre-commit hooks, and +`make check` rebuilds the extension and runs every check without editing a file. +`make install-native` builds with CPU-specific optimizations (not portable). Performance claims +are timed with the A/B/A tool in [dev/perf/](https://github.com/networmix/NetGraph-Core/blob/main/dev/perf/README.md). ## Environment Variables | Variable | Effect | | --- | --- | -| `NGRAPH_CORE_PROFILE=1` | Enable profiling of C++ hot paths (`profiling_dump()` / `profiling_reset()`). | -| `NGRAPH_CORE_BATCH_THREADS` | Worker threads for `batch_max_flow` (default: hardware concurrency). Set to `1` when calling from your own worker pool. | -| `NGRAPH_CORE_SENSITIVITY_THREADS` | Worker threads for `sensitivity_analysis` (default: hardware concurrency). | -| `NGRAPH_CORE_SPF_QUEUE` | Frontier queue for SPF: `heap` forces the reference binary heap, `bucket` prefers the bucket queue where the graph is eligible. Unset (default) selects automatically; both give bit-identical results. Read once per process; for benchmarking and diagnosis. | +| `NGRAPH_CORE_BATCH_THREADS` | Default worker count of `evaluate_pairs`: an integer in [1, 4096]; 1 when unset or invalid, or when a cache is supplied. An explicit `workers` wins. | +| `NGRAPH_CORE_SENSITIVITY_THREADS` | The same for `evaluate_edge_failures`. | ## Requirements @@ -147,4 +127,4 @@ described in [dev/perf/README.md](dev/perf/README.md). ## License -[MIT License](LICENSE) +[MIT License](https://github.com/networmix/NetGraph-Core/blob/main/LICENSE) diff --git a/bindings/python/apply_actions.cpp b/bindings/python/apply_actions.cpp new file mode 100644 index 0000000..eb711f8 --- /dev/null +++ b/bindings/python/apply_actions.cpp @@ -0,0 +1,214 @@ +// Several demands at once: BatchMode, the actions Place, Replace and Forward, and apply_actions. +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/flow_ledger.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "native_call.hpp" +#include "policy_value.hpp" +#include "netgraph/core/detail/policy_engine.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The Python actions keep their arguments as given (the objects themselves, so an action keeps its +// policy and black holes alive); the call converts them. +struct PyPlace { + py::object policy; + py::object offer; +}; +struct PyReplace { + py::object policy; + py::object offer; + py::object design_demand; +}; +struct PyForward { + py::object policy; + py::object offer; + py::object black_holes; +}; + +std::string type_name(const py::handle& obj) { return std::string(py::str(py::type::of(obj).attr("__name__"))); } + +py::object policy_arg(const py::object& policy) { + const auto* type = py::detail::get_type_info(typeid(FlowPolicy), true); + if (PyObject_TypeCheck(policy.ptr(), type->type) == 0) throw py::type_error("policy must be a FlowPolicy (got " + type_name(policy) + ")"); + (void)policy_value(policy); + return policy; +} + +// An offer: an int (an __index__ object is kept as its int) or a float, None refused. +py::object offer_arg(const py::object& offer) { + if (offer.is_none()) throw py::type_error("offer must be an int or a float (got NoneType)"); + return units_as_given(offer, "offer"); +} + +py::object black_holes_arg(const py::object& black_holes) { + (void)eligibility_or_none(black_holes, "black_holes"); + return black_holes; +} + +// The offer of a converted action: +inf nullopt; any value outside the domain -1, so the call's +// own check raises the offer text in its place. +std::optional offer_at_use(const py::object& offer) { return *units_at_use(offer); } + +FlowPolicy& policy_at_use(const py::object& policy) { return policy_value(policy); } + +// The native action of entry i, or the type error of an entry that is not an action. +Action action_of(const py::handle& item, std::size_t i) { + if (py::isinstance(item)) { + const auto& a = item.cast(); + return Place {policy_at_use(a.policy), offer_at_use(a.offer)}; + } + if (py::isinstance(item)) { + const auto& a = item.cast(); + return Replace {policy_at_use(a.policy), offer_at_use(a.offer), units_at_use(a.design_demand)}; + } + if (py::isinstance(item)) { + const auto& a = item.cast(); + const Eligibility* black_holes = a.black_holes.is_none() ? nullptr : a.black_holes.cast(); + return Forward {policy_at_use(a.policy), offer_at_use(a.offer), black_holes}; + } + throw py::type_error("actions[" + std::to_string(i) + "] must be a Place, Replace or Forward (got " + type_name(item) + ")"); +} + +} // namespace + +void register_apply_actions(py::module_& m) { + py::enum_(m, "BatchMode", + "How apply_actions runs its list. ORDERED: each action as its scalar call, in list order, on the\n" + "state the earlier actions left (earlier actions are served first). SIMULTANEOUS: every action forwarded at\n" + "the same time on one view, losses shared on links across the demands.") + .value("ORDERED", BatchMode::Ordered) + .value("SIMULTANEOUS", BatchMode::Simultaneous); + + py::class_(m, "Place", + "An incremental place(offer) of policy in an apply_actions list (ORDERED only). An immutable value:\n" + "policy (a FlowPolicy, kept alive) and offer (int units or a float, kept as given; another type\n" + "raises TypeError here, a value outside the domain raises the call's ValueError at apply_actions).") + .def(py::init([](const py::object& policy, const py::object& offer) { + return PyPlace {policy_arg(policy), offer_arg(offer)}; + }), + py::arg("policy"), py::arg("offer")) + .def_readonly("policy", &PyPlace::policy, "the FlowPolicy") + .def_readonly("offer", &PyPlace::offer, "the offer as given (int units or a float)"); + + py::class_(m, "Replace", + "A replace(offer, design_demand) of policy in an apply_actions list; SIMULTANEOUS takes it only for\n" + "ShortestPaths EQUAL or WEIGHTED and for Pinned. An immutable value: policy (kept alive), offer and\n" + "design_demand (int units or a float, kept as given; None: absent; another type raises TypeError\n" + "here, a value outside the domain raises the call's ValueError at apply_actions).") + .def(py::init([](const py::object& policy, const py::object& offer, const py::object& design_demand) { + return PyReplace {policy_arg(policy), offer_arg(offer), units_as_given(design_demand, "design_demand")}; + }), + py::arg("policy"), py::arg("offer"), py::kw_only(), py::arg("design_demand") = py::none()) + .def_readonly("policy", &PyReplace::policy, "the FlowPolicy") + .def_readonly("offer", &PyReplace::offer, "the offer as given (int units or a float)") + .def_readonly("design_demand", &PyReplace::design_demand, "the design demand as given (int units or a float); None: absent"); + + py::class_(m, "Forward", + "A forward(offer, black_holes) of policy over its installed program in an apply_actions list. An\n" + "immutable value: policy and black_holes (an Eligibility of the policy's graph or None) kept alive,\n" + "offer (int units or a float, kept as given; another type raises TypeError here, a value outside\n" + "the domain raises the call's ValueError at apply_actions).") + .def(py::init([](const py::object& policy, const py::object& offer, const py::object& black_holes) { + return PyForward {policy_arg(policy), offer_arg(offer), black_holes_arg(black_holes)}; + }), + py::arg("policy"), py::arg("offer"), py::kw_only(), py::arg("black_holes") = py::none()) + .def_readonly("policy", &PyForward::policy, "the FlowPolicy") + .def_readonly("offer", &PyForward::offer, "the offer as given (int units or a float)") + .def_readonly("black_holes", &PyForward::black_holes, "the black holes (an Eligibility); None: none"); + + m.def( + "apply_actions", + [](const py::object& ledger, const py::object& actions, const py::object& mode, const py::object& outputs) { + // The arguments' types in argument order, every entry included, before any native check. + if (!py::isinstance(ledger)) throw py::type_error("ledger must be a FlowLedger"); + if (is_invalid_sequence(actions)) { + throw py::type_error("actions must be a sequence of Place, Replace or Forward (got " + type_name(actions) + ")"); + } + const auto items = py::reinterpret_borrow(actions); + const Py_ssize_t count = PySequence_Size(items.ptr()); + if (count < 0) throw py::error_already_set(); + const auto n = static_cast(count); + // The entries are held for the call: the actions borrow their policies and black holes. + std::vector held; + held.reserve(n); + std::vector native; + native.reserve(n); + for (std::size_t i = 0; i < n; ++i) { + PyObject* raw = PySequence_GetItem(items.ptr(), static_cast(i)); + if (raw == nullptr) throw py::error_already_set(); + held.push_back(py::reinterpret_steal(raw)); + native.push_back(action_of(held.back(), i)); + } + if (!py::isinstance(mode)) throw py::type_error("mode must be a BatchMode (got " + type_name(mode) + ")"); + const auto batch_mode = mode.cast(); + const auto o = outputs_of(outputs); + auto& target = ledger.cast(); + detail::check_list_arguments(target, native, batch_mode, o); + // The list and every result holder exist before the native transaction. + auto out = py::reinterpret_steal>(checked(PyList_New(count)).release()); + std::vector slots(n); + for (std::size_t i = 0; i < n; ++i) { + auto [obj, slot] = new_object(); + slots[i] = slot; + PyList_SET_ITEM(out.ptr(), static_cast(i), obj.release().ptr()); + } + native_call([&] { + auto results = apply_actions(target, native, batch_mode, o); + for (std::size_t i = 0; i < n; ++i) *slots[i] = std::move(results[i]); + }); + return out; + }, + py::arg("ledger"), py::arg("actions"), py::kw_only(), py::arg_v("mode", BatchMode::Ordered, "BatchMode.ORDERED"), + py::arg("outputs") = py::none(), + "Run the actions of several policies of ledger (a sequence of Place, Replace and Forward) as one\n" + "transaction -> list[PlacementResult], one per action in input order, each the scalar call's fields\n" + "for its policy. The whole list is validated first, then computed, then published once (one exchange\n" + "of every policy's records, then their epochs): a failed call raises with the ledger and every\n" + "policy unchanged (with the two limits stated in the package docstring). A policy appears\n" + "at most once, and every policy belongs to ledger.\n" + "mode ORDERED: each action runs in list order on a private working state exactly as its scalar call\n" + "would on the state the earlier actions left (their carried load and holds); new records take\n" + "FlowIds in list order. Earlier actions are served first: a reservation gives no protection. mode SIMULTANEOUS:\n" + "Replace of ShortestPaths EQUAL or WEIGHTED (routes prepared on the view) and of Pinned, Forward of\n" + "any policy with an installed program, every one LOSSY with a finite offer, forwarded at the same\n" + "time on one view: the residual plus every participant's old carried load (outside owners and orphans\n" + "stay; holds are never added). At a link that cannot carry its arrivals the loss is shared by highest\n" + "averages over the demands (ties by policy creation order, never by list position), then over each\n" + "demand's sources, then its tunnels; each participant's black holes are its own. Permuting the list\n" + "changes nothing. outputs (OutputOptions; None: none) as for the scalar calls: residual and\n" + "saturated_boundary_edges read the state after that action (ORDERED) or after the whole list\n" + "(SIMULTANEOUS, one residual for every entry, each entry's boundary from its own sources and mask).\n" + "Checked in this order: the arguments' types (TypeError: `ledger must be a FlowLedger`, `actions must\n" + "be a sequence of Place, Replace or Forward (got )`, `actions[] must be a Place, Replace or\n" + "Forward (got )`, `mode must be a BatchMode (got )`, `outputs must be an OutputOptions\n" + "(got )`); mode's value (`mode: is not a value of BatchMode`); then each action's\n" + "argument/immutable checks in list order: `list policy belongs\n" + "to another ledger`, `duplicate policy in list`, SIMULTANEOUS's own rules (`SIMULTANEOUS does not\n" + "accept Place`, `SIMULTANEOUS Replace requires ShortestPaths EQUAL or WEIGHTED, or Pinned`,\n" + "`SIMULTANEOUS requires admission=LOSSY`, `SIMULTANEOUS requires finite offers`), then the scalar\n" + "call's argument checks. Ledger usability and every action's mutable state are then checked under\n" + "the gate, in list order, before any computation. An empty list checks usability and mode, then\n" + "returns []. An error of the computation (a\n" + "loss-iteration error, a solver or forwarding size limit, the record identity limit counted for the\n" + "whole list) publishes nothing."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/concurrency_hooks.cpp b/bindings/python/concurrency_hooks.cpp new file mode 100644 index 0000000..113064f --- /dev/null +++ b/bindings/python/concurrency_hooks.cpp @@ -0,0 +1,198 @@ +// Native rendezvous controls for progress/atomicity tests. Python is called only +// by run(), outside every native lock; the probes themselves touch native state. +#include "registration.hpp" +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/concurrency.hpp" +#include "netgraph/core/detail/concurrency_models.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/flow_ledger.hpp" +#include "netgraph/core/shortest_path_cache.hpp" +#include "native_call.hpp" +#include "values.hpp" + +namespace py = pybind11; +namespace netgraph::core::bindings { +namespace { +using Stage = detail::ConcurrentStage; +constexpr std::array(Stage::Count)> names { + "ledger_wait", "ledger_acquired", "cache_queued", "cache_acquired", + "before_publication", "after_exchange", "native_complete", "worker_begin", "thread_launch", "native_begin", + "cache_tally_complete", "policy_engine_installed", "tunnel_roster_search", "route_definition_check", "route_walk", "argument_begin" +}; +Stage stage_of(const std::string& name) { + for (std::size_t i = 0; i < names.size(); ++i) if (name == names[i]) return static_cast(i); + throw py::value_error("unknown native probe stage: " + name); +} +struct NativeProbe { + std::mutex mutex; + std::condition_variable changed; + std::array counts {}; + std::optional pause, fail; + std::vector failing_workers; + bool reverse_failures {false}; + bool later_failed {false}; + bool released {false}; + int auto_release_ms {0}; + detail::ConcurrentProbe probe {notify, this}; + + NativeProbe(const std::string& paused, const std::string& failed, std::vector workers, bool reverse, + int automatic) + : failing_workers(std::move(workers)), reverse_failures(reverse), auto_release_ms(automatic) { + if (automatic < 0 || automatic > 10000) throw py::value_error("auto_release_ms must be in [0, 10000]"); + if (!paused.empty()) pause = stage_of(paused); + if (!failed.empty()) fail = stage_of(failed); + } + static void notify(void* context, Stage stage, const void* object) { + auto& self = *static_cast(context); + std::unique_lock lock(self.mutex); + ++self.counts[static_cast(stage)]; + self.changed.notify_all(); + if (stage == Stage::WorkerBegin && !self.failing_workers.empty()) { + const auto index = *static_cast(object); + if (std::find(self.failing_workers.begin(), self.failing_workers.end(), index) != self.failing_workers.end()) { + if (self.reverse_failures && index == self.failing_workers.front()) { + if (!self.changed.wait_for(lock, std::chrono::seconds(10), [&] { return self.later_failed; })) { + throw std::runtime_error("worker rendezvous timed out"); + } + } else { + self.later_failed = true; + self.changed.notify_all(); + } + throw std::runtime_error("injected worker failure " + std::to_string(index)); + } + } + if (self.pause == stage && !self.released) { + ++self.counts[names.size()]; + const auto duration = std::chrono::milliseconds(self.auto_release_ms == 0 ? 10000 : self.auto_release_ms); + const bool signalled = self.changed.wait_for(lock, duration, [&] { return self.released; }); + ++self.counts[names.size() + 1]; + if (!signalled && self.auto_release_ms == 0) throw std::runtime_error("native rendezvous timed out"); + self.released = true; + } + if (self.fail == stage) throw std::runtime_error("injected native failure: " + std::string(names[static_cast(stage)])); + } + bool wait(Stage stage, double seconds, std::size_t count) { + std::unique_lock lock(mutex); + return changed.wait_for(lock, std::chrono::duration(seconds), [&] { return counts[static_cast(stage)] >= count; }); + } + void release() { + const std::lock_guard lock(mutex); + released = true; + changed.notify_all(); + } +}; + +// Py_AtExit runs after Python teardown. Keep only native owners here, so this +// fixture cannot accidentally retain a Python global through the daemon frame. +struct ShutdownWaiter { + std::shared_ptr ledger; + std::shared_ptr cache; + std::shared_ptr probe; + std::unique_ptr policy; +}; +std::atomic shutdown_waiter {nullptr}; +void native_shutdown() noexcept { + std::unique_ptr state(shutdown_waiter.exchange(nullptr)); + if (!state) return; + if (Py_IsInitialized() != 0) std::_Exit(91); + { + const std::lock_guard lock(state->probe->mutex); + if (state->probe->counts[names.size()] <= state->probe->counts[names.size() + 1]) std::_Exit(92); + } + std::fputs("CHILD: native finalizer waiting after Python teardown\n", stdout); + std::fflush(stdout); + try { + state->policy.reset(); + if (state->cache) state->cache->thaw(); + state.reset(); + } catch (...) { + std::_Exit(93); + } + std::fputs("CHILD: native finalizer completed\n", stdout); + std::fflush(stdout); +} +} // namespace +void register_concurrency_hooks(py::module_& m) { + py::class_>(m, "_ConcurrencyProbe") + .def(py::init, bool, int>(), + py::arg("pause") = "", py::arg("fail") = "", py::arg("workers") = std::vector{}, py::arg("reverse") = false, + py::arg("auto_release_ms") = 0) + .def("run", [](NativeProbe& self, const py::function& callback, const py::args& args) { + const detail::ScopedConcurrentProbe scope(&self.probe); + return callback(*args); + }, py::arg("callable")) + .def("wait", [](NativeProbe& self, const std::string& stage, double seconds, std::size_t count) { + const auto point = stage_of(stage); + // Do not run native_call's entry probe inside the probe's own wait. + py::gil_scoped_release detached; + return self.wait(point, seconds, count); + }, py::arg("stage"), py::arg("seconds") = 5.0, py::arg("count") = 1) + .def("release", &NativeProbe::release, py::call_guard()) + .def("counts", [](NativeProbe& self) { + std::array snapshot; + { + py::gil_scoped_release detached; + const std::lock_guard lock(self.mutex); + snapshot = self.counts; + } + py::dict out; + for (std::size_t i = 0; i < names.size(); ++i) out[names[i]] = snapshot[i]; + out["pause_begin"] = snapshot[names.size()]; + out["pause_end"] = snapshot[names.size() + 1]; + return out; + }); + m.def("_at_native_exit", [](const std::shared_ptr& ledger, + const std::shared_ptr& cache, const std::shared_ptr& probe) { + if ((!ledger && !cache) || (ledger && cache) || !probe) throw py::value_error("one native exit resource and a probe are required"); + auto state = native_call([&] { + auto s = std::make_unique(); + s->ledger = ledger; + s->cache = cache; + s->probe = probe; + if (ledger) s->policy = std::make_unique(*ledger, std::vector{0}, std::vector{1}, FlowPolicyConfig{}); + return s; + }); + ShutdownWaiter* empty = nullptr; + if (!shutdown_waiter.compare_exchange_strong(empty, state.get())) { + native_call([&] { state.reset(); }); + throw py::value_error("a native exit waiter is already registered"); + } + if (Py_AtExit(native_shutdown) != 0) { + shutdown_waiter.store(nullptr); + native_call([&] { state.reset(); }); + throw std::runtime_error("native exit registration failed"); + } + (void)state.release(); + }); + m.def("_concurrency_model", [](const std::string& name) { + py::dict out; + if (name == "publication-locked" || name == "publication-unlocked") { + const auto result = detail::publication_model(name == "publication-locked"); + out["states"] = result.states; + out["mixed"] = result.mixed; + } else { + const auto result = detail::phase_model(name); + out["states"] = result.states; + out["partial"] = result.partial; + out["python_under_lock"] = result.python_under_lock; + out["late_hits"] = result.late_hits; + } + return out; + }, py::arg("protocol")); +} +} // namespace netgraph::core::bindings +#endif diff --git a/bindings/python/eligibility.cpp b/bindings/python/eligibility.cpp new file mode 100644 index 0000000..6e12c78 --- /dev/null +++ b/bindings/python/eligibility.cpp @@ -0,0 +1,57 @@ +// Eligibility: the static node and edge flags of one graph. +#include +#include +#include + +#include +#include + +#include "netgraph/core/eligibility.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "publication.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_eligibility(py::module_& m) { + py::class_>( + m, "Eligibility", + "Which nodes and edges of one graph may be used: one flag per node and per edge, fixed when built\n" + "(it does not depend on traffic). Built by Graph.eligibility(excluded_nodes, excluded_edges) and\n" + "Eligibility.intersect.") + .def("graph_uid", &Eligibility::graph_uid, "uid of the graph this value belongs to") + .def("uid", &Eligibility::uid, "0 for the all-allowed value, a fresh uid otherwise") + .def( + "node_allowed", + [](const py::object& self) { + const auto& el = self.cast(); + return readonly_view(el.node_allowed(), self, py::dtype("bool")); + }, + "bool[N] read-only view") + .def( + "edge_allowed", + [](const py::object& self) { + const auto& el = self.cast(); + return readonly_view(el.edge_allowed(), self, py::dtype("bool")); + }, + "bool[E] read-only view (not excluded and both endpoints allowed)") + .def_static( + "intersect", + [](const std::shared_ptr& a, const std::shared_ptr& b) { + if (!a || !b) throw py::type_error("Eligibility.intersect takes two Eligibility values"); + if (a->graph_uid() != b->graph_uid()) throw py::value_error("Eligibility.intersect: the two values belong to different graphs"); + auto r = native_call([&] { return Eligibility::intersect(*a, *b); }); + if (r.uid() == a->uid()) return typed_shared(a); + if (r.uid() == b->uid()) return typed_shared(b); + return typed_shared(std::make_shared(std::move(r))); + }, + py::arg("a"), py::arg("b"), + "The intersection of two values of one graph (ValueError for values of different graphs):\n" + "an operand itself when the result equals it (a first, then b), otherwise a new value with a\n" + "fresh uid; equal intersections built separately get distinct uids."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/evaluators.cpp b/bindings/python/evaluators.cpp new file mode 100644 index 0000000..9a6f43c --- /dev/null +++ b/bindings/python/evaluators.cpp @@ -0,0 +1,274 @@ +// evaluate_demand, evaluate_pairs and evaluate_edge_failures with their options and results. Each +// evaluator places on a new, empty ledger; a frozen ShortestPathCache counts its hits only when the whole Python +// call succeeds (detail::frozen_call). +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/shortest_path_cache.hpp" +#include "netgraph/core/detail/evaluator_checks.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/evaluators.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "native_call.hpp" +#include "values.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The Python form of EvaluationOptions: the Eligibility and the cache are held by reference count +// for the call (the C++ structs borrow raw pointers); design_demand and offer as given. +struct PyEvaluationOptions { + FlowPolicyConfig config; + std::shared_ptr eligibility; + std::shared_ptr cache; + OutputOptions outputs; + py::object design_demand; + py::object offer; + [[nodiscard]] EvaluationOptions native() const { + return EvaluationOptions {config, eligibility.get(), cache.get(), outputs, units_at_use(design_demand), units_at_use(offer)}; + } +}; + +// workers: an explicit integer is passed through (validated by the evaluator); None takes the +// environment default (detail::default_workers), and 1 whenever a cache is supplied (a building +// cache has one writer; a frozen one is shared only when the caller asks for workers). +std::int32_t workers_of(const py::object& workers, bool edge_failures, bool cache_given) { + if (!workers.is_none()) { + return static_cast(integer_value(workers, std::numeric_limits::min(), + std::numeric_limits::max(), "workers")); + } + if (cache_given) return 1; + return detail::default_workers(edge_failures); +} + +} // namespace + +void register_evaluators(py::module_& m) { + py::class_(m, "EvaluationOptions", + "The options of every evaluator (evaluate_demand, evaluate_pairs, evaluate_edge_failures): config\n" + "(the configuration), eligibility (the static Eligibility; None: all allowed), cache (the\n" + "ShortestPathCache; None: private per worker), outputs (the optional outputs of each result;\n" + "evaluate_edge_failures: of its baseline), design_demand and\n" + "offer (int units or a float, kept as given; another type raises TypeError). offer: a finite offer\n" + "for every model (None: the uncapped LOSSLESS question, or for ShortestPaths EQUAL/WEIGHTED with\n" + "LOSSY the first-hop experiment; required for DistributedTE, Pinned and, without a finite design,\n" + "CentralizedTE with tunnel_count under LOSSY). design_demand: DistributedTE always, required and\n" + "finite; CentralizedTE with tunnel_count: None means the offer when it is finite, else unlimited;\n" + "every other model leaves it None. The evaluators check both after the configuration, in this order:\n" + "an explicit offer (`offer must be an integer in [0, 2^62)`, +inf included), the design\n" + "(` does not accept design_demand`, `design_demand must be an integer in [0, 2^62) or\n" + "permitted +inf`, `DistributedTE requires a finite design demand`), DistributedTE's missing design\n" + "(`DistributedTE evaluation requires design_demand`), the missing LOSSY offer (` LOSSY evaluation requires a finite offer`).") + .def(py::init([](const FlowPolicyConfig& config, const std::shared_ptr& elig, + const std::shared_ptr& cache, const py::object& outputs, const py::object& design_demand, + const py::object& offer) { + auto d = units_as_given(design_demand, "design_demand"); + auto o = units_as_given(offer, "offer"); + return PyEvaluationOptions {config, elig, cache, outputs_of(outputs), std::move(d), std::move(o)}; + }), + py::arg("config"), py::kw_only(), py::arg("eligibility") = nullptr, py::arg("cache") = nullptr, + py::arg("outputs") = py::none(), py::arg("design_demand") = py::none(), py::arg("offer") = py::none()) + .def_property_readonly("config", field_object(&PyEvaluationOptions::config), + "the FlowPolicyConfig of every evaluated policy (evaluate_edge_failures: the baseline and every deletion)") + .def_property_readonly("eligibility", shared_field(&PyEvaluationOptions::eligibility), "the static Eligibility; None: everything allowed") + .def_property_readonly("cache", shared_field(&PyEvaluationOptions::cache), "the supplied ShortestPathCache; None: a private cache per worker") + .def_property_readonly("outputs", field_object(&PyEvaluationOptions::outputs), + "the OutputOptions of every result (evaluate_edge_failures: of the baseline)") + .def_readonly("design_demand", &PyEvaluationOptions::design_demand, + "the tunnel models' design volume as given (int units or a float); None: absent") + .def_readonly("offer", &PyEvaluationOptions::offer, "the offer as given (int units or a float); None: absent"); + + py::class_(m, "SensitivityEntry", + "One candidate edge: edge and delivery_reduction (int units: the baseline's delivery minus the\n" + "delivery without the edge; signed).") + .def_property_readonly("edge", int_reader(&SensitivityEntry::edge)) + .def_property_readonly("delivery_reduction", int_reader(&SensitivityEntry::delivery_reduction)); + + py::class_(m, "SensitivityResult", + "baseline (the PlacementResult of evaluate_demand()) and entries (ascending EdgeId; an\n" + "absent edge has reduction 0).") + .def_property_readonly("baseline", field_object(&SensitivityResult::baseline)) + .def_property_readonly("entries", list_field(&SensitivityResult::entries)); + + m.def( + "evaluate_demand", + [](const std::shared_ptr& graph, const py::object& sources, const py::object& destinations, + const py::object& opts) { + if (!graph) throw py::type_error("graph must be a Graph"); + const auto o = options_of(opts, "EvaluationOptions"); + const auto s = node_ids(sources, graph->num_nodes(), "sources"); + const auto d = node_ids(destinations, graph->num_nodes(), "destinations"); + const EvaluationOptions c = o.native(); + const auto roster = detail::check_demand_arguments(*graph, s, d, c, detached_argument_work); + auto [obj, slot] = new_object(); + native_call([&] { + return detail::frozen_call(c.cache, [&](detail::FrozenCacheHitTally& tally) { + *slot = detail::evaluate_demand_in_call(*graph, s, d, c, tally, roster ? &*roster : nullptr); + return true; + }); + }); + return std::move(obj); + }, + py::arg("graph"), py::arg("sources"), py::arg("destinations"), py::arg("options"), + "The evaluation of a fresh FlowPolicy with options.config on a fresh ledger of graph at options.offer\n" + "(None: +inf; DistributedTE and CentralizedTE with tunnel_count: evaluate(offer, design_demand=...))\n" + "-> PlacementResult. LOSSLESS delivers up to a finite offer; LOSSY accounts for exactly that offer\n" + "(ShortestPaths EQUAL/WEIGHTED LOSSY without an offer: the first-hop experiment). The options are checked as\n" + "the EvaluationOptions docstring states.\n" + "Without an offer, CentralizedTE(source_allocation=OPTIMIZED) gives the exact maximum flow (a finite offer below\n" + "it stops the solve there, with no cut); with source_allocation=EQUAL, the largest\n" + "total whose round-robin source shares are all carried; with WEIGHTED (CentralizedTE without tunnel_count),\n" + "the largest total whose highest-averages shares of source_weights are all carried.\n" + "A supplied frozen cache raises ShortestPathCacheMiss\n" + "for an absent key, unchanged.\n" + "A failed call raises; a supplied ShortestPathCache then behaves as the ShortestPathCache docstring states."); + + m.def( + "evaluate_pairs", + [](const std::shared_ptr& graph, const py::object& pairs, const py::object& opts, + const py::object& per_pair_eligibility, const py::object& workers) { + if (!graph) throw py::type_error("graph must be a Graph"); + // The arguments first, then each pair completely and in pair order (the same two parts + // as the C++ validation), so the lowest failing pair index is reported. + const auto o = options_of(opts, "EvaluationOptions"); + const auto w = workers_of(workers, false, static_cast(o.cache)); + const EvaluationOptions c = o.native(); + detail::check_batch_arguments(*graph, c, w); + if (is_invalid_sequence(pairs)) throw py::type_error("pairs must be a sequence of (sources, destinations)"); + const auto items = py::reinterpret_borrow(pairs); + const auto num_pairs = py::len(items); + std::optional eligs; + if (!per_pair_eligibility.is_none()) { + // An explicit sequence has one entry per pair (None entries allowed), also when empty. + if (is_invalid_sequence(per_pair_eligibility)) throw py::type_error("per_pair_eligibility must be a sequence of Eligibility or None"); + eligs = py::reinterpret_borrow(per_pair_eligibility); + if (py::len(*eligs) != num_pairs) { + throw py::value_error("per_pair_eligibility has " + std::to_string(py::len(*eligs)) + " entries for " + + std::to_string(num_pairs) + " pairs"); + } + } + std::vector, std::vector>> ps; + std::vector> keep; + std::vector per; + std::exception_ptr pending; + const auto n = graph->num_nodes(); + for (std::size_t i = 0; i < num_pairs; ++i) { + try { + const py::object item = items[i]; + // Not a sequence is a wrong type, a sequence of another length a wrong value: one text. + if (is_invalid_sequence(item)) throw py::type_error("each pair must be (sources, destinations)"); + if (py::len(item) != 2) throw py::value_error("each pair must be (sources, destinations)"); + auto t = py::reinterpret_borrow(item); + auto src = node_ids(t[0], n, "sources"); + auto dst = node_ids(t[1], n, "destinations"); + const Eligibility* e = nullptr; + if (eligs) { + const py::object entry = (*eligs)[i]; + if (!entry.is_none()) { + if (!py::isinstance(entry)) throw py::type_error("per_pair_eligibility entries must be Eligibility or None"); + keep.push_back(entry.cast>()); + e = keep.back().get(); + } + per.push_back(e); + } + detail::check_batch_pair(*graph, c.config, src, dst, e, i); + ps.emplace_back(std::move(src), std::move(dst)); + } catch (...) { + pending = std::current_exception(); + break; + } + } + const auto rosters = detail::prepare_batch_rosters(*graph, ps, c, per, w, detached_argument_work); + if (pending) std::rethrow_exception(pending); + auto out = py::reinterpret_steal(checked(PyList_New(static_cast(num_pairs))).release()); + std::vector slots(num_pairs); + for (std::size_t i = 0; i < num_pairs; ++i) { + auto [obj, slot] = new_object(); + slots[i] = slot; + PyList_SET_ITEM(out.ptr(), static_cast(i), obj.release().ptr()); + } + native_call([&] { + return detail::frozen_call(c.cache, [&](detail::FrozenCacheHitTally& tally) { + auto rs = detail::evaluate_pairs_in_call(*graph, ps, c, per, w, tally, rosters); + for (std::size_t i = 0; i < rs.size(); ++i) *slots[i] = std::move(rs[i]); + return true; + }); + }); + return out; + }, + py::arg("graph"), py::arg("pairs"), py::arg("options"), py::arg("per_pair_eligibility") = py::none(), + py::arg("workers") = py::none(), + "evaluate_demand() for each (sources, destinations) pair, results in pair order. Pair i uses\n" + "options.eligibility " + "intersected with per_pair_eligibility[i] (None entries allowed; a given sequence has one entry per pair,\n" + "also when empty). workers >= 1; None: NGRAPH_CORE_BATCH_THREADS or 1, and 1 when a cache is\n" + "supplied. All checks finish before evaluation workers start: the arguments first, then each pair\n" + "completely in pair order, so the error of the lowest failing pair is raised. First-pass tunnel\n" + "searches run in parallel under one GIL release, before any lease or gate. Pinned is rejected\n" + "(`evaluate_pairs does not accept Pinned`: a program has fixed endpoints);\n" + "an empty batch returns [].\n" + "Caches: a supplied building cache needs workers == 1; a frozen cache is shared by every worker\n" + "(pass workers explicitly), and its ShortestPathCacheMiss reaches the caller unchanged (the lowest failing\n" + "pair's). An intersection that differs from both operands gets a new Eligibility uid, which a\n" + "frozen cache cannot hold: pass the combined per-pair values with eligibility None, or a common value\n" + "that contains them.\n" + "A failed call raises; a supplied ShortestPathCache then behaves as the ShortestPathCache docstring states."); + + m.def( + "evaluate_edge_failures", + [](const std::shared_ptr& graph, const py::object& sources, const py::object& destinations, + const py::object& opts, const py::object& workers) { + if (!graph) throw py::type_error("graph must be a Graph"); + const auto o = options_of(opts, "EvaluationOptions"); + const auto s = node_ids(sources, graph->num_nodes(), "sources"); + const auto d = node_ids(destinations, graph->num_nodes(), "destinations"); + const auto w = workers_of(workers, true, static_cast(o.cache)); + const EvaluationOptions c = o.native(); + const auto roster = detail::check_edge_failure_arguments(*graph, s, d, c, w, detached_argument_work); + auto [obj, slot] = new_object(); + native_call([&] { + return detail::frozen_call(c.cache, [&](detail::FrozenCacheHitTally& tally) { + *slot = detail::evaluate_edge_failures_in_call(*graph, s, d, c, w, tally, roster ? &*roster : nullptr); + return true; + }); + }); + return std::move(obj); + }, + py::arg("graph"), py::arg("sources"), py::arg("destinations"), py::arg("options"), py::arg("workers") = py::none(), + "The baseline evaluate_demand() and one SensitivityEntry per candidate edge. Candidates: with unrestricted\n" + "CentralizedTE and the optimized source allocation, the edges that carry flow in the baseline (removing any\n" + "other edge keeps the baseline feasible, at any offer); otherwise every allowed edge, whatever its capacity (an edge of capacity 0\n" + "can carry a route, so deleting it can change the result, and with the equal source allocation a deletion that\n" + "removes a source from the roster can raise the total: a negative reduction). delivery_reduction =\n" + "baseline.call_delivered minus the call_delivered of a fresh evaluate_demand() without the edge, at the\n" + "same offer, design and other options, in signed units, never thresholded; it is not the traffic the\n" + "edge carried. ShortestPaths\n" + "gives reduction 0 without placing to an edge outside the static shortest-path support reachable from\n" + "the roster sources (no route can use it). CentralizedTE with a support solves every deletion afresh; Pinned\n" + "replays the same program under each deletion. workers as evaluate_pairs\n" + "(NGRAPH_CORE_SENSITIVITY_THREADS). Only ShortestPaths reads the cache, and each placed deletion uses a\n" + "new Eligibility uid, so a frozen cache raises ShortestPathCacheMiss there.\n" + "A failed deletion fails the call (the lowest failing candidate's exception, no partial result); a\n" + "supplied ShortestPathCache then behaves as the ShortestPathCache docstring states."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_ledger.cpp b/bindings/python/flow_ledger.cpp new file mode 100644 index 0000000..d60ddb8 --- /dev/null +++ b/bindings/python/flow_ledger.cpp @@ -0,0 +1,136 @@ +// FlowLedger: the integer ledger of one graph. +#include +#include +#include + +#include + +#include "netgraph/core/flow_ledger.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "values.hpp" +#include "netgraph/core/detail/ledger_access.hpp" +#include "netgraph/core/detail/endpoints.hpp" +#include "native_call.hpp" +#include "publication.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_flow_ledger(py::module_& m) { + [[maybe_unused]] auto ledger_class = py::class_>( + m, "FlowLedger", + "The integer ledger of one graph, in two projections that are never added: carried traffic (the\n" + "service baseline; carried() is the sum of every record's edge_flows and residual() = baseline -\n" + "carried()) and the strict holds of T1 tunnels (the reservation baseline; reserved() is the sum of\n" + "every record's edge_reservations and unreserved() = reservation baseline - reserved()), all int64[E]\n" + "snapshots. Traffic may use bandwidth that is held: holds never police. The baselines are replaced\n" + "only by reset(); the records are the flows placed on it by policies (the ledger has no public\n" + "placement, removal or reservation). reset() and copy() are rejected while a policy is bound (a\n" + "FlowPolicy built on this ledger still exists).") + .def(py::init([](const std::shared_ptr& graph, const py::object& service_baseline, const py::object& reservation_baseline) { + if (!graph) throw py::type_error("graph must be a Graph"); + const auto caps = graph->edge_capacity_units(); + const auto r = service_baseline.is_none() ? std::vector(caps.begin(), caps.end()) + : edge_units(service_baseline, *graph, "service_baseline"); + const auto u = reservation_baseline.is_none() ? std::vector(caps.begin(), caps.end()) + : edge_units(reservation_baseline, *graph, "reservation_baseline"); + return native_call([&] { return std::make_shared(graph, r, u); }); + }), + py::arg("graph"), py::arg("service_baseline") = py::none(), py::kw_only(), py::arg("reservation_baseline") = py::none(), + "service_baseline and reservation_baseline: the baselines as integer units, one entry per edge in\n" + "[0, the edge's capacity] (a physical baseline goes through to_units first; service_baseline\n" + "checked first); None: the capacities. A wrong type (not a 1-D sequence of integers, a non-integer\n" + "entry) raises TypeError, a wrong length or an entry outside its range ValueError.") + .def("uid", [](const FlowLedger& l) { return native_call([&] { return l.uid(); }); }, "process-unique ledger uid (the first half of every FlowId)") + .def( + "graph", [](const FlowLedger& fg) { auto graph = native_call([&] { return std::shared_ptr(fg.graph()); }); return typed_shared(std::move(graph)); }, + "the ledger's graph") + .def( + "edge_capacity_units", + [](const py::object& self) { + const auto& ledger = self.cast(); + const auto capacities = native_call([&] { return ledger.edge_capacity_units(); }); + return readonly_view(capacities, self); + }, + "int64[E] the graph's capacities in units (read-only view)") + .def( + "residual", [](const FlowLedger& fg) { return int64_array(native_call([&] { return fg.residual(); })); }, + "int64[E] snapshot: service_baseline - carried()") + .def( + "carried", [](const FlowLedger& fg) { return int64_array(native_call([&] { return fg.carried(); })); }, + "int64[E] snapshot: the sum of every record's edge_flows") + .def( + "reserved", [](const FlowLedger& fg) { return int64_array(native_call([&] { return fg.reserved(); })); }, + "int64[E] snapshot: strict holds per edge (T1), the sum of every record's edge_reservations") + .def( + "unreserved", [](const FlowLedger& fg) { return int64_array(native_call([&] { return fg.unreserved(); })); }, + "int64[E] snapshot: reservation baseline minus strict holds, per edge") + .def( + "records", [](const FlowLedger& fg) { return list_of(native_call([&] { return fg.records(); })); }, + "live records as snapshots in FlowId order: those of bound policies and orphans (records left by a\n" + "policy destroyed without remove())") + .def( + "record", + [](const FlowLedger& fg, const py::object& flow_id) { const auto id = flow_id_of(flow_id); return typed_object(native_call([&] { return fg.record(id); })); }, + py::arg("flow_id"), + "one record by FlowId (ValueError for a FlowId of another ledger, a retired or an unknown id)") + .def( + "saturated_boundary_edges", + [](const FlowLedger& fg, const py::object& sources, const std::shared_ptr& elig) { + const auto s = node_ids(sources, detail::LedgerAccess::graph(fg)->num_nodes(), "sources"); + if (elig && elig->graph_uid() != detail::LedgerAccess::graph(fg)->uid()) throw py::value_error("foreign Eligibility: built for another graph"); + return native_call([&] { return fg.saturated_boundary_edges(s, elig.get()); }); + }, + py::arg("sources"), py::arg("eligibility") = nullptr, + "Saturation report: the eligible edges with capacity >= 1 unit that leave the set of nodes the\n" + "allowed sources can reach (following eligible edges forward with residual >= 1 unit), in\n" + "ascending EdgeId. It ignores the destinations, so it is not a source-destination cut.") + .def( + "reset", + [](FlowLedger& fg, const py::object& service_baseline, const py::object& reservation_baseline) { + const auto& graph = detail::LedgerAccess::graph(fg); + const auto caps = graph->edge_capacity_units(); + const auto r = service_baseline.is_none() ? std::vector(caps.begin(), caps.end()) + : edge_units(service_baseline, *graph, "service_baseline"); + const auto u = reservation_baseline.is_none() ? std::vector(caps.begin(), caps.end()) + : edge_units(reservation_baseline, *graph, "reservation_baseline"); + native_call([&] { fg.reset(r, u); }); + }, + py::arg("service_baseline") = py::none(), py::kw_only(), py::arg("reservation_baseline") = py::none(), + "New baselines (None: the capacities); orphaned records with their allocations and holds are\n" + "cleared and their ids retired (ids are never reused). ValueError while a policy is bound.") + .def( + "copy", [](const FlowLedger& fg) { return native_call([&] { return std::make_shared(fg); }); }, + "a new ledger (new uid) with the same baselines, records and holds, ids kept under the new uid;\n" + "ValueError while a policy is bound") + .def("__copy__", [](const FlowLedger& fg) { return native_call([&] { return std::make_shared(fg); }); }) + .def("__deepcopy__", [](const FlowLedger& fg, const py::object&) { return native_call([&] { return std::make_shared(fg); }); }, + py::arg("memo")); +#if NETGRAPH_CORE_PRIVATE_HOOKS + ledger_class + .def( + "_state", + [](const FlowLedger& fg) { + struct Snapshot { + std::int64_t next; + std::vector bound, orphans; + }; + const auto state = native_call([&] { + const auto gate = detail::LedgerAccess::lock(fg); + return Snapshot {detail::LedgerAccess::next_record_counter(fg), + detail::LedgerAccess::bound_policy_ids(fg), detail::LedgerAccess::orphan_policy_ids(fg)}; + }); + py::dict out; + out["next_flow_id"] = state.next; + out["bound"] = state.bound; + out["orphans"] = state.orphans; + return out; + }, + "private: {next_flow_id, bound, orphans} (owner ids)"); +#endif +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_policy.cpp b/bindings/python/flow_policy.cpp new file mode 100644 index 0000000..88522a5 --- /dev/null +++ b/bindings/python/flow_policy.cpp @@ -0,0 +1,387 @@ +// FlowPolicy: the one placement engine, bound to its ledger. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/shortest_path_cache.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/flow_ledger.hpp" +#include "netgraph/core/detail/policy_engine.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/forwarding_program.hpp" +#include "integers.hpp" +#include "publication.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "policy_value.hpp" +#include "netgraph/core/detail/ledger_access.hpp" +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include + +#include "netgraph/core/detail/route_builders.hpp" +#include "netgraph/core/detail/ledger_access.hpp" +#include "netgraph/core/detail/policy_engine.hpp" +#include "netgraph/core/detail/private_hooks.hpp" +#include "trace.hpp" +#endif +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// Register all dependencies before binding the policy. This leaves no flag or +// incremented reference behind when +// a map/vector allocation fails. with_internals supplies pybind11's registry +// lock on a free-threaded interpreter; no Python callback runs inside it. +void pin_policy(py::detail::instance* nurse, std::initializer_list dependencies) { + std::vector pins; + pins.reserve(dependencies.size()); + for (const auto dependency : dependencies) { + if (!dependency.is_none()) pins.push_back(dependency.ptr()); + } + py::detail::with_internals([&](py::detail::internals& internals) { + const auto [it, inserted] = internals.patients.emplace(reinterpret_cast(nurse), std::move(pins)); + (void)it; + if (!inserted) throw std::logic_error("FlowPolicy dependencies already registered"); + }); + // Arguments still pin these objects. The remaining steps cannot fail and run + // after the registry lock, before this shell can publish a bound policy. + for (const auto dependency : dependencies) { + if (!dependency.is_none()) Py_INCREF(dependency.ptr()); + } + nurse->has_patients = true; +} + +// A request or a full offer: units in [0, 2^62) or +inf (float('inf')); nullopt is +inf. +std::optional request_units(const py::object& offer) { + return units_or_inf(offer, "offer must be an integer in [0, 2^62) or permitted +inf"); +} + +// A design demand argument: None absent; float('inf') the inner nullopt; an integer passed on for +// the call's own checks in their order (one outside int64 as -1, outside the domain like it); any +// other value raises the domain text here (a conversion failure comes first). +std::optional> design_units(const py::object& design) { + constexpr const char* text = "design_demand must be an integer in [0, 2^62) or permitted +inf"; + if (design.is_none()) return std::nullopt; + if (is_bool(design) || !PyIndex_Check(design.ptr())) return units_or_inf(design, text); + const py::object idx = python_int(design, "design_demand"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + return std::optional(overflow != 0 ? Amount {-1} : static_cast(v)); +} + +#if NETGRAPH_CORE_PRIVATE_HOOKS + +// The epoch state of a policy (roster, requested, +inf, per roster source the flows). +py::dict state_obj(const py::object& self) { + const auto& policy = policy_value(self); + const auto& pe = detail::PolicyAccess::engine(policy); + struct Flow { detail::LiveFlow flow; FlowRecord record; }; + struct Snapshot { + std::vector roster; + Counter128 requested; + bool infinite; + Amount delivered {0}; + std::vector> record_groups; + }; + auto snapshot = native_call([&] { + const auto gate = detail::LedgerAccess::lock(pe.ledger); + Snapshot state {pe.reachable_sources, pe.epoch.requested, pe.epoch.infinite, 0, {}}; + state.record_groups.resize(pe.epoch.record_groups.size()); + for (std::size_t i = 0; i < state.record_groups.size(); ++i) { + for (const auto& flow : pe.epoch.record_groups[i]) { + auto record = detail::LedgerAccess::live(pe.ledger, flow.record_counter); + state.delivered += record.delivered; + state.record_groups[i].push_back({flow, std::move(record)}); + } + } + return state; + }); + py::dict out; + const auto& r = pe.config.routing; + out["mode"] = detail::model_name(r); + out["roster"] = snapshot.roster; + out["requested"] = counter_int(snapshot.requested); + out["infinite"] = snapshot.infinite; + out["placed_demand"] = snapshot.delivered; + py::dict record_groups; + // Per roster source its flows (T1, T2 and Pinned: per program entry, keyed by its position). + const auto* gl = std::get_if(&r); + const bool per_entry = std::holds_alternative(r) || std::holds_alternative(r) || + (gl != nullptr && gl->tunnel_count.has_value()); + for (std::size_t i = 0; i < snapshot.record_groups.size(); ++i) { + py::list flows; + for (const auto& row : snapshot.record_groups[i]) { + const auto& s = row.flow; + const FlowRecord& rec = row.record; + py::dict f; + f["flow_id"] = py::make_tuple(rec.flow_id.first, rec.flow_id.second); + f["src"] = s.source; + const auto edges = detail::PredecessorDAGBuilder::ascending(*s.route); + f["route"] = std::vector(edges.begin(), edges.end()); + f["T"] = rec.delivered; + f["delivered"] = rec.delivered; + f["alloc"] = pairs_list(rec.edge_flows); + f["entry"] = s.entry_id < 0 ? py::object(py::none()) : py::object(py::int_(s.entry_id)); + flows.append(f); + } + record_groups[py::int_(per_entry ? static_cast(i) : static_cast(snapshot.roster[i]))] = flows; + } + out["cohorts"] = record_groups; + return out; +} + +#endif + +} // namespace + +// A policy's destructor waits for its ledger gate: it runs detached, as native work does. +struct DetachedDelete { + void operator()(FlowPolicy* policy) const noexcept { + const Detached detached; + delete policy; + } +}; + +void register_flow_policy(py::module_& m) { + [[maybe_unused]] auto flow_policy = py::class_>( + m, "FlowPolicy", py::is_final(), + "The placement engine for one demand, bound to its ledger at construction. A final Python class.\n" + "Native computation releases the GIL; one ledger gate serializes calls and mutable reads. Every call (place,\n" + "evaluate, replace, forward) is all-or-nothing: a failed call raises and leaves the ledger and the\n" + "policy as they were (with the two limits stated in the package docstring). remove()\n" + "takes every flow of the policy back out, clears its program and starts a new epoch (the request\n" + "total starts again from 0); replace and forward start one too. A policy dropped without remove()\n" + "leaves its records in the ledger as orphans: readable, never changed again, cleared by\n" + "FlowLedger.reset(). sources and destinations are one NodeId or a sequence; the two sets must not\n" + "overlap. ValueError for invalid endpoints, configurations and requests and for objects of another\n" + "graph, ledger or policy; TypeError for a ledger (None included), config, eligibility, cache or\n" + "outputs of the wrong class, a node id or a request of a wrong type (a bool, a float node id, a\n" + "string) and an unknown keyword.") + .def("__init__", [](py::detail::value_and_holder& instance, const std::shared_ptr& ledger, + const py::object& sources, const py::object& destinations, const FlowPolicyConfig& config, + const std::shared_ptr& elig, const std::shared_ptr& cache) { + if (!ledger) throw py::type_error("ledger must be a FlowLedger"); + const auto n = detail::LedgerAccess::graph(*ledger)->num_nodes(); + const auto s = node_ids(sources, n, "sources"); + const auto d = node_ids(destinations, n, "destinations"); + auto prepared = detail::PolicyAccess::prepare(*ledger, s, d, config, elig.get(), cache.get(), nullptr, detached_argument_work); + // Registration and holder construction can allocate. Complete them while the + // shell is unbound; no Python work remains after native construction publishes. + std::unique_ptr holder(detail::PolicyAccess::shell().release()); + auto* policy = holder.get(); + instance.inst->owned = false; + instance.value_ptr() = policy; + try { + instance.type->init_instance(instance.inst, &holder); + instance.inst->owned = true; + } catch (...) { + instance.value_ptr() = nullptr; + throw; + } + try { + pin_policy(instance.inst, {py::cast(ledger), py::cast(elig), py::cast(cache)}); + native_call([&] { detail::PolicyAccess::construct(*policy, std::move(prepared), cache.get()); }); + } catch (...) { + // A failed __init__ must leave no usable shell, including explicit __new__ + // followed by a caught constructor failure. Final classes also forbid a + // subclass that swallows __init__'s exception. + if (instance.instance_registered()) { + py::detail::deregister_instance(instance.inst, instance.value_ptr(), instance.type); + instance.set_instance_registered(false); + } + instance.type->dealloc(instance); + if (instance.inst->has_patients) py::detail::clear_patients(reinterpret_cast(instance.inst)); + throw; + } + }, py::detail::is_new_style_constructor(), + py::arg("ledger"), + py::arg("sources"), py::arg("destinations"), py::arg("config"), py::arg("eligibility") = nullptr, + py::arg("cache") = nullptr, + "Bind a policy to ledger. eligibility (None: everything allowed) and cache (None: none; only ShortestPaths\n" + "routing reads it, another model rejects it) belong to the ledger's graph; the policy keeps the\n" + "ledger, eligibility and cache alive (a Pinned policy shares its program and black holes). The roster\n" + "(the sources that statically reach an allowed destination, over allowed edges whatever their\n" + "capacity, the tunnel models' heads; Pinned: the program's heads that reach an allowed destination over its basis\n" + "under eligibility) is fixed here. Everything is validated before the policy binds, in this order: the endpoints, the\n" + "graph of eligibility, the program, the support, the cache and the black holes (`foreign Eligibility:\n" + "built for another graph`, `program belongs to another graph`, `support belongs to another graph`,\n" + "`foreign ShortestPathCache: bound to another graph`, `black_holes belongs to another graph`), the\n" + "configuration (FlowPolicyConfig's order, the pooled single-path rule and T2's several-sources rule\n" + "included), the tunnel limit (`tunnel limit exceeded:\n" + " heads x tunnels > 1073741823`) and a Pinned program's endpoints (`program endpoints do not\n" + "match policy endpoints`).") + .def( + "place", + [](const py::object& self, const py::object& offer, const py::object& outputs) { + auto& p = policy_value(self); + // The operation comes before the offer (the call's order). + detail::check_place_operation(detail::PolicyAccess::engine(p)); + const auto v = request_units(offer); + const auto o = outputs_of(outputs); + detail::check_policy_arguments(detail::PolicyAccess::engine(p), detail::PolicyOperation::Place, v); + auto [obj, slot] = new_object(); + native_call([&] { *slot = p.place(v, o); }); + return std::move(obj); + }, + py::arg("offer"), py::arg("outputs") = py::none(), + "Place offer (int units in [0, 2^62), or float('inf')) all or nothing -> PlacementResult. T1 and T2\n" + "raise ` does not support incremental placement`\n" + "(also for 0).\n" + "Requests add up within an epoch (from construction, the last remove(), replace() or forward());\n" + "after float('inf') any finite request is rejected, 0 included. A request of 0 places nothing and\n" + "reports the current state. ShortestPaths prepares its program at the epoch's first positive request.\n" + "LOSSLESS admits the largest total in [admitted, admitted + offer] that the program carries with no\n" + "loss (a refused unit is not offered again); LOSSY forwards the new round-robin (or captured-weight)\n" + "injections, each head splitting its increment afresh (earlier drops are not offered again); a\n" + "controller re-solves admitted + offer. A LOSSY float('inf') is ShortestPaths's first-hop experiment;\n" + "Pinned raises `LOSSY forwarding requires a finite offer`. outputs (OutputOptions; None: none)\n" + "requests edge_flows, residual, saturated_boundary_edges, edge_reservations and min_cut. Everything is\n" + "validated before any change; a\n" + "failed call raises with the ledger and the policy unchanged. A solve of the flow core (CentralizedTE\n" + "or ShortestPaths OPTIMIZED) on a network beyond its int32 numbering raises ValueError:\n" + "more than INT32_MAX - 2 nodes, or more than 2^30 - 1 edges, sources and destinations together. A\n" + "LOSSY forwarding whose losses wait on each other in a cycle raises ValueError when its rounds\n" + "alternate between two states or reach the round limit, RuntimeError when the result's check fails.") + .def("remove", [](const py::object& self) { auto& p = policy_value(self); native_call([&] { p.remove(); }); }, + "revert every flow and hold of the policy, retire the ids, clear the program, open a new epoch") + .def("delivered", [](const py::object& self) { auto& p = policy_value(self); return native_call([&] { return p.delivered(); }); }, "the sum of delivered over the policy's live records") + .def( + "records", [](const py::object& self) { auto& p = policy_value(self); return list_of(native_call([&] { return p.records(); })); }, + "the policy's live records as snapshots in FlowId order") + .def( + "record", + [](const py::object& self, const py::object& flow_id) { + auto& p = policy_value(self); const auto id = flow_id_of(flow_id); return typed_object(native_call([&] { return p.record(id); })); }, + py::arg("flow_id"), + "one of the policy's live records by FlowId (ValueError for a FlowId of another ledger or policy, a\n" + "retired or an unknown id)") + .def( + "evaluate", + [](const py::object& self, const py::object& offer, const py::object& design_demand, const py::object& outputs) { + auto& p = policy_value(self); + const auto v = request_units(offer); + const auto d = design_units(design_demand); + const auto o = outputs_of(outputs); + detail::check_policy_arguments(detail::PolicyAccess::engine(p), detail::PolicyOperation::Evaluate, v, d); + auto [obj, slot] = new_object(); + native_call([&] { *slot = p.evaluate(v, d, o); }); + return std::move(obj); + }, + py::arg("offer"), py::kw_only(), py::arg("design_demand") = py::none(), py::arg("outputs") = py::none(), + "A full, independent computation and forwarding of offer (int units or float('inf')) from the\n" + "policy's own released view (the residual plus its own carried load; T1 sets up on the unreserved\n" + "capacity with its own holds released; T2 lays out on the released residual), as replace would do it -> PlacementResult (the computed\n" + "program and plan, the flags replace would set, the outputs of the evaluated state); publishes\n" + "nothing, creates no record, books no hold and leaves the epoch as it was. design_demand as for\n" + "replace. LOSSY requires a finite offer.") + .def( + "replace", + [](const py::object& self, const py::object& offer, const py::object& design_demand, const py::object& outputs) { + auto& p = policy_value(self); + const auto v = request_units(offer); + const auto d = design_units(design_demand); + const auto o = outputs_of(outputs); + detail::check_policy_arguments(detail::PolicyAccess::engine(p), detail::PolicyOperation::Replace, v, d); + auto [obj, slot] = new_object(); + native_call([&] { *slot = p.replace(v, d, o); }); + return std::move(obj); + }, + py::arg("offer"), py::kw_only(), py::arg("design_demand") = py::none(), py::arg("outputs") = py::none(), + "Recompute from the policy's own released view and forward the full offer, published atomically ->\n" + "PlacementResult: the old records and holds are retired (their ids never live again), new ones\n" + "created, and a new epoch starts at this offer (delivery may go down: delivery_delta). ShortestPaths\n" + "prepares its routes again on that view; a\n" + "controller re-solves; T1 sets its tunnels up again for the design and books their holds; T2 lays its\n" + "tunnels out again for the design (planned bandwidth, never held);\n" + "Pinned forwards its program. design_demand (T1 and T2: int units, or float('inf'); None: equal to\n" + "the offer): T1 needs a finite design (`DistributedTE requires a finite design demand`), T2 takes an unlimited\n" + "one (its largest common size); every other model raises ` does not accept design_demand`.\n" + "In this order: the offer, LOSSY's finite offer, then the design demand (the model, its domain,\n" + "finiteness).") + .def( + "forward", + [](const py::object& self, const py::object& offer, const py::object& black_holes, const py::object& outputs) { + auto& p = policy_value(self); + // Only the immutable operation check precedes offer conversion. + detail::check_forward_operation(detail::PolicyAccess::engine(p)); + const auto v = request_units(offer); + const auto bh = eligibility_or_none(black_holes, "black_holes"); + const auto o = outputs_of(outputs); + detail::check_policy_arguments(detail::PolicyAccess::engine(p), detail::PolicyOperation::Forward, v, {}, bh.get()); + auto [obj, slot] = new_object(); + native_call([&] { *slot = p.forward(v, bh.get(), o); }); + return std::move(obj); + }, + py::arg("offer"), py::kw_only(), py::arg("black_holes") = py::none(), py::arg("outputs") = py::none(), + "Forward a full new offer over the installed program, with no route computation and no setup, on the\n" + "released view -> PlacementResult; a new epoch like replace, the program, its plan and T1's holds\n" + "kept (no flag; T1's records are recreated with the same edge_reservations). LOSSLESS admits the\n" + "program's lossless bound up to the offer; LOSSY sends the offer. black_holes (an Eligibility of the\n" + "graph): the nodes and edges it excludes are black holes for this call (stale forwarding: traffic\n" + "reaching one is dropped there; a black-holed source loses its share). In this order: Pinned raises\n" + "`Pinned forwards with replace`; the offer; LOSSY +inf `LOSSY forwarding requires a finite offer`;\n" + "black_holes of another graph `black_holes belongs to another graph`; then, under the gate,\n" + "no installed program `forward requires an installed program`.") + .def_property_readonly( + + "program", + [](const py::object& self) { auto& p = policy_value(self); return optional_shared(native_call([&] { return p.program(); })); }, + "the installed ForwardingProgram; None before the first preparation and after remove()") + .def_property_readonly("config", [](const py::object& self) { auto& p = policy_value(self); return typed_object(p.config()); }, + "the configuration as validated at construction, ShortestPaths' absent\n" + "source_allocation and weight_basis resolved to the allocation's defaults"); +#if NETGRAPH_CORE_PRIVATE_HOOKS + py::class_(m, "_PlacementTrace") + .def("as_dict", [](const detail::Trace& trace) { return trace_obj(trace); }, + "private: convert the completed native trace to Python values"); + flow_policy + .def("_seed_requested", [](const py::object& self, std::uint64_t hi, std::uint64_t lo) { + auto& p = policy_value(self); + if (hi >= (std::uint64_t{1} << 63)) throw py::value_error("requested seed reaches 2^127"); + native_call([&] { + auto& engine = detail::PolicyAccess::engine(p); + const auto gate = detail::LedgerAccess::lock(engine.ledger); + engine.epoch.requested = Counter128 {hi, lo}; + }); + }, py::arg("hi"), py::arg("lo"), "private: seed the exact requested counter") + .def( + "_place_traced", + [](const py::object& self, const py::object& offer, const py::object& outputs) { + auto& p = policy_value(self); + detail::check_place_operation(detail::PolicyAccess::engine(p)); + const auto v = request_units(offer); + const auto o = outputs_of(outputs); + detail::check_policy_arguments(detail::PolicyAccess::engine(p), detail::PolicyOperation::Place, v); + auto [obj, slot] = new_object(); + auto [trace_obj, trace] = new_object(); + auto pair = py::reinterpret_steal(checked(PyTuple_New(2)).release()); + PyTuple_SET_ITEM(pair.ptr(), 0, obj.release().ptr()); + PyTuple_SET_ITEM(pair.ptr(), 1, trace_obj.release().ptr()); + native_call([&] { *slot = detail::place_traced(p, v, o, *trace); }); + return pair; + }, + py::arg("offer"), py::arg("outputs") = py::none(), + "private: place() with the transaction's trace (the call's intermediate values, keyed as\n" + "the tests' model of the policy keys them)") + .def("_state", &state_obj, "private: the epoch state (roster, requested, +inf, record_groups)") + .def( + "_owner", [](const py::object& self) { auto& p = policy_value(self); return detail::PolicyAccess::engine(p).policy_id; }, + "private: the ledger-local owner id (FlowRecord.policy_id)"); +#endif +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_policy_config.cpp b/bindings/python/flow_policy_config.cpp new file mode 100644 index 0000000..8c41a92 --- /dev/null +++ b/bindings/python/flow_policy_config.cpp @@ -0,0 +1,97 @@ +// FlowPolicyConfig and OutputOptions: the configuration of a placement. +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/options.hpp" +#include "integers.hpp" +#include "strict_bool.hpp" +#include "publication.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +Routing routing_of(const py::object& obj) { + if (py::isinstance(obj)) return obj.cast(); + if (py::isinstance(obj)) return obj.cast(); + if (py::isinstance(obj)) return obj.cast(); + if (py::isinstance(obj)) return obj.cast(); + throw py::type_error("routing must be ShortestPaths, DistributedTE, CentralizedTE or Pinned (got " + + std::string(py::str(py::type::of(obj).attr("__name__"))) + ")"); +} + +// The routing as a new Python object through the checked allocation (publication.hpp): a failed +// allocation raises instead of crashing. +py::object routing_obj(const Routing& r) { + return std::visit([](const auto& x) { return object_of(x); }, r); +} + +} // namespace + +void register_policy_config(py::module_& m) { + py::class_( + m, "FlowPolicyConfig", + "One demand's settings: the routing model, which holds every option it reads (ShortestPaths,\n" + "DistributedTE (T1), CentralizedTE, CentralizedTE(tunnel_count=...) (T2) or Pinned), and the admission\n" + "(LOSSLESS by default; LOSSY for ShortestPaths EQUAL and WEIGHTED, T1, T2 and Pinned). The\n" + "combination is checked when a policy or an evaluator uses the configuration (ValueError), in\n" + "this order: the enumeration values; the model matrix (route and source allocation, admission,\n" + "the ineffective tie rule, the pooled single-path rule, prefer_headroom, the weight combinations,\n" + "and last a ShortestPathCache on a model that does not read it); the tunnel count, an up_to count\n" + "for DistributedTE, the ceilings, min_unreserved_bandwidth and edge_weights' length and entries;\n" + "the tunnel limit (roster heads times the tunnel count at most 2^30 - 1); a Pinned program's\n" + "endpoints.") + .def(py::init([](const py::object& routing, Admission admission) { + return FlowPolicyConfig {routing_of(routing), admission}; + }), + py::kw_only(), py::arg("routing"), py::arg_v("admission", Admission::Lossless, "Admission.LOSSLESS")) + .def_property_readonly( + "routing", [](const FlowPolicyConfig& c) { return routing_obj(c.routing); }, + "the routing model: ShortestPaths, DistributedTE, CentralizedTE or Pinned") + .def_property_readonly("admission", field_object(&FlowPolicyConfig::admission), "LOSSLESS or LOSSY") + .def("__repr__", [](const FlowPolicyConfig& c) { + return "FlowPolicyConfig(routing=" + own_repr(routing_obj(c.routing)) + ", admission=" + + (c.admission == Admission::Lossy ? "Admission.LOSSY" : "Admission.LOSSLESS") + ")"; + }); + + py::class_(m, "OutputOptions", + "Optional outputs of a placement (all false by default; absent outputs are None):\n" + "edge_flows (the policy's own allocation per edge after the call), residual and\n" + "saturated_boundary_edges (the ledger's after the call), edge_reservations (the policy's own\n" + "strict holds per edge after the call; zeros for a model without holds), and min_cut (unrestricted\n" + "CentralizedTE with the optimized source allocation, when the call's solve reached the maximum\n" + "flow; [] for an empty roster, place(0) excepted; None for every other model or source allocation,\n" + "for place(0) and for a forward with a nonempty roster).") + .def(py::init([](StrictBool edge_flows, StrictBool residual, StrictBool saturated_boundary_edges, StrictBool edge_reservations, + StrictBool min_cut) { + OutputOptions o; + o.edge_flows = edge_flows; + o.residual = residual; + o.saturated_boundary_edges = saturated_boundary_edges; + o.edge_reservations = edge_reservations; + o.min_cut = min_cut; + return o; + }), + py::kw_only(), py::arg("edge_flows") = StrictBool {false}, py::arg("residual") = StrictBool {false}, + py::arg("saturated_boundary_edges") = StrictBool {false}, py::arg("edge_reservations") = StrictBool {false}, + py::arg("min_cut") = StrictBool {false}) + .def_readonly("edge_flows", &OutputOptions::edge_flows, "request PlacementResult.edge_flows") + .def_readonly("residual", &OutputOptions::residual, "request PlacementResult.residual") + .def_readonly("saturated_boundary_edges", &OutputOptions::saturated_boundary_edges, "request PlacementResult.saturated_boundary_edges") + .def_readonly("edge_reservations", &OutputOptions::edge_reservations, + "request PlacementResult.edge_reservations") + .def_readonly("min_cut", &OutputOptions::min_cut, "request PlacementResult.min_cut"); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_record.cpp b/bindings/python/flow_record.cpp new file mode 100644 index 0000000..8e6aff0 --- /dev/null +++ b/bindings/python/flow_record.cpp @@ -0,0 +1,84 @@ +// FlowRecord: one flow of a ledger, as a snapshot. +#include +#include +#include + +#include +#include + +#include "netgraph/core/flow_record.hpp" +#include "netgraph/core/route_structures.hpp" +#include "values.hpp" +#include "publication.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +py::object cost_or_none(Cost c) { return c == kUnreachable ? py::object(py::none()) : py::object(int_object(c)); } + +} // namespace + +void register_flow_record(py::module_& m) { + py::class_( + m, "FlowRecord", + "One flow of a ledger, as a snapshot: flow_id (ledger uid, id), policy_id (the policy's id),\n" + "sources, destinations, admission, entry_id (a T1, T2 or Pinned policy's program entry, else None),\n" + "route (PredecessorDAG: the flow's route), min_route_cost (its minimum walk cost; None when empty),\n" + "offered (Lossy only, else None), delivered, edge_flows [(EdgeId, units)] (carried load) and\n" + "edge_drops [(EdgeId, units)] (cumulative, ascending EdgeId), edge_reservations [(EdgeId, units)]\n" + "(T1's strict holds, ascending EdgeId). offered and the edge_drops units are exact Python ints (they\n" + "may exceed 64 bits).") + .def_property_readonly( + + "flow_id", [](const FlowRecord& r) { return int_pair(r.flow_id.first, r.flow_id.second); }, + "(ledger uid, id): ids are assigned in creation order and never reused in the ledger") + .def_property_readonly("policy_id", int_reader(&FlowRecord::policy_id), "the ledger-local id of the policy that created the flow") + .def_property_readonly( + "sources", [](const FlowRecord& r) { return ints_list(std::span(r.sources)); }, + "the flow's source node ids, ascending") + .def_property_readonly( + "destinations", [](const FlowRecord& r) { return ints_list(std::span(r.destinations)); }, + "the flow's destination node ids, ascending") + .def_property_readonly("admission", field_object(&FlowRecord::admission), "LOSSLESS or LOSSY") + .def_property_readonly("entry_id", optional_int_reader(&FlowRecord::entry_id), + "the program entry the record carries (T1, T2 and Pinned: the entry identifier); None for the\n" + "records kept per roster source (ShortestPaths, CentralizedTE)") + .def_property_readonly( + "route", [](const FlowRecord& r) { return typed_shared(r.route); }, + "the flow's route (an immutable PredecessorDAG shared with the ledger)") + .def_property_readonly( + "min_route_cost", [](const FlowRecord& r) { return cost_or_none(r.min_route_cost); }, + "the cost of the cheapest walk of the route from the sources to the destinations; None when\n" + "the route has no such walk") + .def_property_readonly( + + "offered", + [](const FlowRecord& r) -> py::object { + return r.offered.has_value() ? counter_int(*r.offered) : py::object(py::none()); + }, + "Lossy: the units offered to the flow, summed over calls (an exact int, may exceed 64 bits);\n" + "None for a Lossless flow") + .def_property_readonly("delivered", int_reader(&FlowRecord::delivered), "the units the flow delivers to its destinations (its total)") + .def_property_readonly( + "edge_flows", [](const FlowRecord& r) { return pairs_list(r.edge_flows); }, + "the flow's allocation [(EdgeId, units)], positive entries in ascending EdgeId") + .def_property_readonly( + "edge_drops", + [](const FlowRecord& r) { return pairs_list(r.edge_drops); }, + "Lossy: the units dropped per edge [(EdgeId, units)], summed over calls, positive entries in\n" + "ascending EdgeId (exact ints); empty for a Lossless flow") + .def_property_readonly( + "edge_reservations", [](const FlowRecord& r) { return pairs_list(r.edge_reservations); }, + "T1: the strict holds of an up tunnel [(EdgeId, units)], its request on every link of its path,\n" + "ascending EdgeId; empty for every other record (holds never police traffic)") + .def("__repr__", [](const FlowRecord& r) { + return "FlowRecord(flow_id=(" + std::to_string(r.flow_id.first) + ", " + std::to_string(r.flow_id.second) + + "), policy_id=" + std::to_string(r.policy_id) + ", delivered=" + std::to_string(r.delivered) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/flow_solver_hook.cpp b/bindings/python/flow_solver_hook.cpp new file mode 100644 index 0000000..2b62104 --- /dev/null +++ b/bindings/python/flow_solver_hook.cpp @@ -0,0 +1,109 @@ +// The private entry _solve_min_cost_flow, registered in both builds: the flow core's +// minimum-cost flow from zero on a given view, with its full payload for the flow checks. +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/min_cost_flow.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "registration.hpp" +#include "native_call.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +py::list int_list(const std::vector& v) { + py::list out(v.size()); + for (std::size_t i = 0; i < v.size(); ++i) out[i] = py::int_(v[i]); + return out; +} + +} // namespace + +void register_flow_solver_hook(py::module_& m) { + m.def( + "_solve_min_cost_flow", + [](const std::shared_ptr& graph, const py::object& view, const py::object& sources, + const py::object& destinations, const std::shared_ptr& elig, + const py::object& cap) -> py::dict { + if (!graph) throw py::type_error("graph must be a Graph"); + // The core's order (view length, view entries, sources, destinations, overlap, cap, + // Eligibility), every value checked before narrowing; the core re-checks the same order. + const auto v = edge_amounts(view, *graph, "view", "view (capacity units)"); + const auto s = checked_ids(wide_integers(sources, "sources node ids"), graph->num_nodes(), "sources"); + const auto d = checked_ids(wide_integers(destinations, "destinations node ids"), graph->num_nodes(), "destinations"); + detail::check_disjoint(detail::node_set(s, graph->num_nodes(), "sources"), + detail::node_set(d, graph->num_nodes(), "destinations"), "sources", "destinations"); + std::optional c; + if (!cap.is_none()) { + if (is_bool(cap) || !PyIndex_Check(cap.ptr())) { + throw py::type_error("cap must be an integer number of units in [0, 2^63 - 2] or None"); + } + py::object idx = python_int(cap, "cap"); + int overflow = 0; + const long long value = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || value < 0 || value > detail::kMaxFlowCap) { + throw py::value_error("cap " + std::string(py::str(idx)) + " is outside [0, 2^63 - 2]"); + } + c = static_cast(value); + } + if (elig && elig->graph_uid() != graph->uid()) { + throw py::value_error("foreign Eligibility: built for another graph"); + } + const auto view_units = v.narrow(); + auto r = native_call([&] { return detail::solve_min_cost_flow(*graph, view_units, s, d, elig.get(), c); }); + py::dict out; + out["flow_value"] = py::int_(r.flow_value); + py::list curve; + for (const auto& [cost, units] : r.marginal_cost_curve) { + py::list entry; + entry.append(py::int_(cost)); + entry.append(py::int_(units)); + curve.append(entry); + } + out["marginal_cost_curve"] = curve; + out["edge_flows"] = int_list(r.edge_flows); + if (r.min_cut.has_value()) { + py::list cut; + for (EdgeId e : *r.min_cut) cut.append(py::int_(e)); + out["min_cut"] = cut; + } else { + out["min_cut"] = py::none(); + } + out["potentials"] = int_list(r.potentials); + py::dict counters; + counters["dijkstra_runs"] = py::int_(r.counters.dijkstra_runs); + counters["dijkstra_pops"] = py::int_(r.counters.dijkstra_pops); + counters["settled"] = py::int_(r.counters.settled); + counters["arc_scans"] = py::int_(r.counters.arc_scans); + counters["dinic_phases"] = py::int_(r.counters.dinic_phases); + counters["augmentations"] = py::int_(r.counters.augmentations); + out["counters"] = counters; + return out; + }, + py::arg("graph"), py::arg("view"), py::arg("sources"), py::arg("destinations"), py::arg("eligibility"), + py::arg("cap"), + "Private flow-core entry: the largest flow up to cap (None: no cap) from the source set to the\n" + "destination set over the eligible edges, sent at the lowest total cost, on the capacities `view`\n" + "(int64 units, EdgeId order), starting from zero flow. Returns {flow_value, marginal_cost_curve:\n" + "[[cost, units]], edge_flows, min_cut (None while the destinations are still reachable, because\n" + "the cap stopped the solve, and on cap 0), potentials (per-node prices from the shortest-path runs\n" + "that prove the flow is cheapest; -1 = never reached), counters}. Invalid inputs raise ValueError\n" + "before any work. A solve (cap not 0) then raises ValueError, before the solver sizes anything, on\n" + "a network beyond the flow core's int32 numbering: more than INT32_MAX - 2 nodes, or more than\n" + "2^30 - 1 eligible edges, eligible sources and eligible destinations together."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/forwarding_program.cpp b/bindings/python/forwarding_program.cpp new file mode 100644 index 0000000..7634bd0 --- /dev/null +++ b/bindings/python/forwarding_program.cpp @@ -0,0 +1,262 @@ +// Forwarding programs: TunnelStatus, Tunnel, RouteDefinition and ForwardingProgram with from_routes and from_dag. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/program_data.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/route_structures.hpp" +#include "netgraph/core/forwarding_program.hpp" +#include "integers.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "publication.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +py::typing::Union route_obj(const RouteDefinition& r) { + py::object out = std::holds_alternative>(r.route) + ? shared_object(std::get>(r.route)) + : shared_object(std::get>(r.route)); + return py::reinterpret_steal>(out.release()); +} + + +// A per-edge integer argument of from_routes read without narrowing (its type defects raise +// here); its values are checked later, in the order of the call's validation. +std::optional wide_or_none(const py::object& obj, const char* name) { + if (obj.is_none()) return std::nullopt; + (void)sequence_length(obj, name); + return wide_integers(obj, name); +} + +// Validate and narrow captured weights at the factory's weight-validation step. +std::vector checked_edge_weights(const CheckedIntegerVector& weights, std::size_t num_edges) { + if (weights.size() != num_edges) throw py::value_error("edge_weights: expected " + std::to_string(num_edges) + " entries"); + for (std::size_t e = 0; e < num_edges; ++e) { + if (!weights.in(e, 0, kDomainLimit - 1)) { + throw py::value_error("edge_weights[" + std::to_string(e) + "] must be an integer in [0, 2^62)"); + } + } + return weights.narrow(); +} + +} // namespace + +void register_program(py::module_& m) { + py::enum_(m, "TunnelStatus", + "A tunnel's state. UP; INSUFFICIENT_BANDWIDTH and COST_CEILING_EXCEEDED: T1's setup left it down (no path\n" + "passes the bandwidth filter, or the cheapest exceeds the cost ceiling): no path, weight 0, and it\n" + "stays down in every forwarding of its program; and in one forwarding only (a by_tunnel row)\n" + "FAILED: its path touches a node or an edge the forwarding policy's Eligibility excludes, so it\n" + "gets nothing and its head splits over its other entries (replay), and BLACKHOLED: up, but its\n" + "path reaches a black hole of stale forwarding, where its traffic is dropped.") + .value("UP", TunnelStatus::Up) + .value("INSUFFICIENT_BANDWIDTH", TunnelStatus::InsufficientBandwidth) + .value("COST_CEILING_EXCEEDED", TunnelStatus::CostCeilingExceeded) + .value("FAILED", TunnelStatus::Failed) + .value("BLACKHOLED", TunnelStatus::Blackholed); + + py::class_(m, "Tunnel", + "One tunnel (a path entry) of a ForwardingProgram, read-only. entry_id: the entry identifier (T1 and\n" + "T2: j*H + i for round j and roster position i, T2's dead heads leaving holes; CentralizedTE and ShortestPaths\n" + "OPTIMIZED: the path's position in the decomposition order; supplied routes: the route's position in\n" + "the list), never a position in program.tunnels; head; local_index: its position among the head's entries of both kinds;\n" + "destination: where its path ends; edges: its EdgeIds in path order; path_cost: the sum of their\n" + "costs (all three None for a tunnel that is down); status: UP, INSUFFICIENT_BANDWIDTH or\n" + "COST_CEILING_EXCEEDED; requested_bandwidth and reserved_bandwidth: T1's strict bandwidth (its request,\n" + "and what it holds on every link of its path: the request when UP, else 0; 0 for the other models);\n" + "planned_bandwidth: T2's planned bandwidth\n" + "u (metadata, never held; 0 for the other models); weight: its integer share among the head's\n" + "entries (T1: 1 up, 0 down; T2: 1; a controller path: its solved amount).") + .def_property_readonly("entry_id", int_reader(&Tunnel::entry_id)) + .def_property_readonly("head", int_reader(&Tunnel::head)) + .def_property_readonly("local_index", int_reader(&Tunnel::local_index)) + .def_property_readonly("destination", optional_int_reader(&Tunnel::destination)) + .def_property_readonly( + + "edges", + [](const Tunnel& t) -> py::object { + return t.edges.has_value() ? py::object(ints_tuple(*t.edges)) : py::object(py::none()); + }, + "the path's EdgeIds in path order; None for a tunnel that is down") + .def_property_readonly("path_cost", optional_int_reader(&Tunnel::path_cost)) + .def_property_readonly("status", [](const Tunnel& t) { return typed_object(t.status); }) + .def_property_readonly("requested_bandwidth", int_reader(&Tunnel::requested_bandwidth)) + .def_property_readonly("reserved_bandwidth", int_reader(&Tunnel::reserved_bandwidth)) + .def_property_readonly("planned_bandwidth", int_reader(&Tunnel::planned_bandwidth)) + .def_property_readonly("weight", int_reader(&Tunnel::weight)) + .def("__repr__", [](const Tunnel& t) { + return "Tunnel(entry_id=" + std::to_string(t.entry_id) + ", head=" + std::to_string(t.head) + + ", weight=" + std::to_string(t.weight) + ")"; + }); + + py::class_(m, "RouteDefinition", + "A supplied route for ForwardingProgram.from_routes: a Path with one edge per hop (a tunnel) or a\n" + "PredecessorDAG (a split route) starting at `head`; weight is its integer share among the head's routes\n" + "(0 allowed: it carries nothing). from_routes validates it.") + .def(py::init([](const py::object& head, const py::object& route, const py::object& weight) { + RouteDefinition r; + r.head = static_cast( + integer_value(head, std::numeric_limits::min(), std::numeric_limits::max(), "head")); + if (py::isinstance(route)) { + r.route = std::shared_ptr(route.cast>()); + } else if (py::isinstance(route)) { + r.route = std::shared_ptr(route.cast>()); + } else { + throw py::type_error("RouteDefinition route must be a Path or a PredecessorDAG (got " + + std::string(py::str(py::type::of(route).attr("__name__"))) + ")"); + } + r.weight = integer_value(weight, std::numeric_limits::min(), std::numeric_limits::max(), "weight"); + return r; + }), + py::arg("head"), py::arg("route"), py::kw_only(), py::arg("weight") = 1, + "RouteDefinition(head, route, *, weight=1): head a NodeId, route a Path or a PredecessorDAG (TypeError\n" + "otherwise), weight an integer (checked in [0, 2^62) by from_routes).") + .def_property_readonly("head", int_reader(&RouteDefinition::head), "the route's head (a source of the program)") + .def_property_readonly("route", &route_obj, "the Path (a tunnel) or the PredecessorDAG (a split route)") + .def_property_readonly("weight", int_reader(&RouteDefinition::weight), "the route's integer share among the head's routes") + .def("__repr__", [](const RouteDefinition& r) { + return "RouteDefinition(head=" + std::to_string(r.head) + ", weight=" + std::to_string(r.weight) + ")"; + }); + + py::class_>( + m, "ForwardingProgram", + "An immutable value: one demand's routes and integer weights. sources and destinations (the normalized\n" + "sets); tunnels: the path entries in ascending entry_id (split routes are not listed); source_weights: the\n" + "captured weights ((source, weight) for every roster source, ascending): optimized controllers' solved\n" + "supplies, weighted CentralizedTE's declared weights, or None for equal sources (round robin). Each head splits its share by\n" + "highest averages over its entries' weights; identical routes are never merged. ShortestPaths and CentralizedTE\n" + "produce one (FlowPolicy.program, PlacementResult.program), and so do T1 and T2 (their tunnels);\n" + "from_routes builds one from given routes and from_dag one shared hop-by-hop program on a DAG; Pinned\n" + "forwards one. No public constructor.") + .def_property_readonly( + + "sources", [](const ForwardingProgram& p) { return ints_tuple(p.sources()); }, "the source set, ascending") + .def_property_readonly( + + "destinations", [](const ForwardingProgram& p) { return ints_tuple(p.destinations()); }, + "the destination set, ascending") + .def_property_readonly( + + "tunnels", + [](const ForwardingProgram& p) { + const auto t = p.tunnels(); + return tuple_of(std::vector(t.begin(), t.end())); + }, + "the program's tunnels (Tunnel), ascending entry_id") + .def_property_readonly( + + "source_weights", + [](const ForwardingProgram& p) -> py::object { + const auto& w = p.source_weights(); + if (!w.has_value()) return py::none(); + return pairs_tuple(*w); + }, + "the captured source weights ((source, weight), ascending), or None for equal sources") + .def_static( + "from_routes", + [](const std::shared_ptr& graph, const py::object& sources, const py::object& destinations, + const py::object& routes, std::optional allocation, std::optional weight_basis, + const py::object& edge_weights, const py::object& residual) { + if (!graph) throw py::type_error("graph must be a Graph"); + // Conversion first (TypeError and integer type defects), then the validation order of + // from_routes; the values beyond int64 are checked here in that order. + if (is_invalid_sequence(routes)) throw py::type_error("routes must be a sequence of RouteDefinition"); + std::vector rs; + for (const auto& item : py::reinterpret_borrow(routes)) { + if (!py::isinstance(item)) throw py::type_error("routes must be a sequence of RouteDefinition"); + rs.push_back(item.cast()); + } + const auto ew = wide_or_none(edge_weights, "edge_weights"); + const auto rw = wide_or_none(residual, "residual"); + const auto n = graph->num_nodes(); + const auto s = detail::node_set(node_ids(sources, n, "sources"), n, "sources"); + const auto d = detail::node_set(node_ids(destinations, n, "destinations"), n, "destinations"); + detail::check_disjoint(s, d, "sources", "destinations"); + if (rs.empty()) throw py::value_error("routes must not be empty"); + bool any_dag = false; + for (const auto& r : rs) any_dag = any_dag || std::holds_alternative>(r.route); + detail::check_route_options("from_routes", any_dag, allocation, weight_basis, ew.has_value(), rw.has_value()); + RouteOptions options; + options.allocation = allocation; + options.weight_basis = weight_basis; + if (ew.has_value()) options.edge_weights = checked_edge_weights(*ew, static_cast(graph->num_edges())); + if (rw.has_value()) options.residual = edge_amounts(residual, *graph, "residual", "residual (units)").narrow(); + auto checked = native_call([&] { return detail::check_route_definitions(*graph, s, d, rs, options); }); + return typed_shared(native_call([&] { + auto prepared = std::move(checked); + return detail::ProgramAccess::from_checked_routes(s, d, rs, std::move(prepared)); + })); + }, + py::arg("graph"), py::arg("sources"), py::arg("destinations"), py::arg("routes"), py::kw_only(), + py::arg("allocation") = py::none(), py::arg("weight_basis") = py::none(), py::arg("edge_weights") = py::none(), + py::arg("residual") = py::none(), + "Validate and freeze supplied routes -> ForwardingProgram. A wrong type raises TypeError; structural,\n" + "range and combination defects raise ValueError, in this order: the endpoint sets; `routes must not be empty`; when no route is a PredecessorDAG, any option given\n" + "(` applies only to PredecessorDAG routes`); allocation\n" + "OPTIMIZED (`from_routes allocation must be EQUAL or WEIGHTED`); weight_basis only with allocation=WEIGHTED,\n" + "EXPLICIT with edge_weights, RESIDUAL_SNAPSHOT with residual; edge_weights' length and entries in [0,\n" + "2^62) and residual's (one entry per edge in [0, capacity]); then each route in list order: its\n" + "route's graph, its head a source, a Path with one edge per hop that starts at the head, ends at a\n" + "destination and meets none before, a PredecessorDAG with a walk from the head to a destination, its weight\n" + "in [0, 2^62). Order and duplicates are kept; a route of weight 0 is validated in full. allocation\n" + "(None: EQUAL) is the interior split of the PredecessorDAG routes; each split route is its own\n" + "split context (a router splits only that route's share). The roster is the sources with a route.") + .def_static( + "from_dag", + [](const std::shared_ptr& graph, const py::object& sources, const py::object& destinations, + const py::object& dag, std::optional allocation, std::optional weight_basis, + const py::object& edge_weights, const py::object& residual) { + if (!graph) throw py::type_error("graph must be a Graph"); + // Conversion first (TypeError and integer type defects), then the validation order of + // from_dag; the values beyond int64 are checked here in that order. + if (!py::isinstance(dag)) { + throw py::type_error("dag must be a PredecessorDAG (got " + std::string(py::str(py::type::of(dag).attr("__name__"))) + + ")"); + } + const auto shared_dag = dag.cast>(); + const auto ew = wide_or_none(edge_weights, "edge_weights"); + const auto rw = wide_or_none(residual, "residual"); + const auto n = graph->num_nodes(); + const auto s = detail::node_set(node_ids(sources, n, "sources"), n, "sources"); + const auto d = detail::node_set(node_ids(destinations, n, "destinations"), n, "destinations"); + detail::check_disjoint(s, d, "sources", "destinations"); + if (shared_dag->graph_uid() != graph->uid()) throw py::value_error("dag belongs to another graph"); + detail::check_route_options("from_dag", true, allocation, weight_basis, ew.has_value(), rw.has_value()); + RouteOptions options; + options.allocation = allocation; + options.weight_basis = weight_basis; + if (ew.has_value()) options.edge_weights = checked_edge_weights(*ew, static_cast(graph->num_edges())); + if (rw.has_value()) options.residual = edge_amounts(residual, *graph, "residual", "residual (units)").narrow(); + return typed_shared(native_call([&] { return ForwardingProgram::from_dag(*graph, s, d, shared_dag, options); })); + }, + py::arg("graph"), py::arg("sources"), py::arg("destinations"), py::arg("dag"), py::kw_only(), + py::arg("allocation") = py::none(), py::arg("weight_basis") = py::none(), py::arg("edge_weights") = py::none(), + py::arg("residual") = py::none(), + "One shared hop-by-hop program on a supplied PredecessorDAG of the graph -> ForwardingProgram: the sources\n" + "allocated equally and pooled at every shared router exactly as ShortestPaths EQUAL/WEIGHTED (one\n" + "split context), the DAG trimmed to the walks from each source to its first destinations. A wrong type\n" + "raises TypeError; structural, range and combination defects raise ValueError, in this order: the endpoint sets; `dag belongs to another graph`; allocation OPTIMIZED\n" + "(`from_dag allocation must be EQUAL or WEIGHTED`); weight_basis only with allocation=WEIGHTED, EXPLICIT\n" + "with edge_weights, RESIDUAL_SNAPSHOT with residual; edge_weights' length and entries in [0, 2^62) and\n" + "residual's (one entry per edge in [0, capacity]). A dag of another type raises TypeError. The roster is\n" + "the sources that reach a destination in the DAG (possibly none): one split route per roster source,\n" + "identified by roster position, no tunnel; a replay rebuilds the roster on the DAG."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/graph.cpp b/bindings/python/graph.cpp new file mode 100644 index 0000000..77f2920 --- /dev/null +++ b/bindings/python/graph.cpp @@ -0,0 +1,221 @@ +// Graph: the immutable graph on integer units and its validating constructor. +#include +#include +#include + +#include +#include +#include +#include + +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "integers.hpp" +#include "strict_float.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The unit-value step on the capacity argument whatever its shape (the shape step comes next): +// every integer entry it holds must lie in [0, 2^62). A NumPy array is read flattened; nested +// entries of a Python sequence and a non-integer non-sequence are left to the shape step. +void check_unit_values(const py::handle& edge_capacity_units) { + const auto out_of_domain = [](const std::string& what, const std::string& value) { + return py::value_error(what + " = " + value + " is outside the units domain [0, 2^62)"); + }; + if (py::isinstance(edge_capacity_units)) { + py::object flat = edge_capacity_units.attr("ravel")(); + const auto w = wide_integers(flat, "edge_capacity_units"); + for (std::size_t i = 0; i < w.size(); ++i) { + if (!w.in(i, 0, kDomainLimit - 1)) throw out_of_domain("edge_capacity_units[" + std::to_string(i) + "]", w.text(i)); + } + return; + } + if (!is_invalid_sequence(edge_capacity_units)) { + auto seq = py::reinterpret_borrow(edge_capacity_units); + for (std::size_t i = 0; i < seq.size(); ++i) { + py::object item = seq[i]; + if (is_nested(item)) continue; + const std::string what = "edge_capacity_units[" + std::to_string(i) + "]"; + py::object idx = python_int(item, what); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= kDomainLimit) throw out_of_domain(what, std::string(py::str(idx))); + } + return; + } + if (!is_bool(edge_capacity_units) && PyIndex_Check(edge_capacity_units.ptr())) { + py::object idx = python_int(edge_capacity_units, "edge_capacity_units"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= kDomainLimit) throw out_of_domain("edge_capacity_units", std::string(py::str(idx))); + } +} + +std::shared_ptr make_graph(const py::object& num_nodes, const py::object& source, + const py::object& destination, const py::object& edge_capacity_units, + const py::object& cost, StrictFloat resolution) { + // The constructor's validation order: the resolution, the unit values, the shapes, zero or + // negative costs, the remaining bounds (ids in range, then the sums); every value is read + // without narrowing, and the messages are the C++ constructor's, which re-checks the same order. + (void)resolution_exponent(resolution); + check_unit_values(edge_capacity_units); + // The shapes: num_nodes, then every array argument rank 1 (in signature order), then equal + // lengths; no entry of source, destination or cost is read before the cost step. + const auto n = integer_value(num_nodes, 0, std::numeric_limits::max(), "num_nodes"); + const std::size_t m = sequence_length(source, "edge_sources"); + const std::size_t md = sequence_length(destination, "edge_destinations"); + const std::size_t mu = sequence_length(edge_capacity_units, "edge_capacity_units"); + const std::size_t mc = sequence_length(cost, "edge_costs"); + if (md != m || mu != m || mc != m) { + throw py::value_error("edge_sources, edge_destinations, edge_capacity_units and edge_costs must have the same length (got " + + std::to_string(m) + ", " + std::to_string(md) + ", " + std::to_string(mu) + ", " + + std::to_string(mc) + ")"); + } + const auto cap = wide_integers(edge_capacity_units, "edge_capacity_units"); // rank 1, values checked + if (m > static_cast(std::numeric_limits::max())) { + throw py::value_error("the number of edges exceeds INT32_MAX (array length)"); + } + const auto w = wide_integers(cost, "edge_costs"); + for (std::size_t i = 0; i < m; ++i) { + if (w.sign(i) < 0 || (w.sign(i) == 0 && w.value[i] < 1)) { + throw py::value_error("edge_costs[" + std::to_string(i) + "] = " + w.text(i) + ": costs must be positive (edge_costs >= 1)"); + } + } + const auto s = wide_integers(source, "edge_sources"); + const auto d = wide_integers(destination, "edge_destinations"); + for (std::size_t i = 0; i < m; ++i) { + if (!s.in(i, 0, n - 1) || !d.in(i, 0, n - 1)) { + throw py::value_error("edge " + std::to_string(i) + ": endpoint (" + s.text(i) + ", " + d.text(i) + + ") out of node range [0, " + std::to_string(n) + ")"); + } + } + constexpr auto kLimit = static_cast(kDomainLimit); + std::uint64_t cap_sum = 0; + std::uint64_t cost_sum = 0; + for (std::size_t i = 0; i < m; ++i) { + cap_sum += static_cast(cap.value[i]); // each below 2^62: no wrap before the check + if (cap_sum >= kLimit) { + throw py::value_error("the sum of all capacities reaches the bound of the units domain (2^62 units)"); + } + if (w.sign(i) > 0 || static_cast(w.value[i]) >= kLimit - cost_sum) { + throw py::value_error("the sum of all edge costs reaches the bound 2^62: path costs would overflow int64"); + } + cost_sum += static_cast(w.value[i]); + } + auto cap_v = cap.narrow(); + auto cost_v = w.narrow(); + auto src_v = s.narrow(); + auto dst_v = d.narrow(); + return native_call([&] { return std::make_shared(static_cast(n), src_v, dst_v, cap_v, cost_v, resolution); }); +} + +} // namespace + +void register_graph(py::module_& m) { + py::class_>( + m, "Graph", + "Immutable directed multigraph on integer units (parallel edges and self-loops allowed). Edges are\n" + "given in input order and numbered by sorting on (source, cost, destination), ties kept in input\n" + "order: that number is the public EdgeId. input_positions()[e] is the input position of edge e.\n" + "Per-edge values use EdgeId order; incoming_edges() instead lists EdgeIds in incoming-slot order.\n" + "Costs are positive integers; capacities are int64 units of the explicit resolution q (a power of\n" + "two in [2^-32, 2^32]); physical capacities enter through to_units.") + .def(py::init(&make_graph), py::arg("num_nodes"), py::arg("edge_sources"), py::arg("edge_destinations"), + py::arg("edge_capacity_units"), py::arg("edge_costs"), py::arg("resolution"), + "Validates, in order, and reports the first defect: the resolution (a power of two in\n" + "[2^-32, 2^32]; a bool is a TypeError), the unit values (every capacity in [0, 2^62)), the array\n" + "shapes, zero or negative costs, the remaining bounds (endpoints in range, then the capacity\n" + "and cost sums below 2^62); a value of a wrong type (a bool or a float for an integer, a\n" + "non-sequence, an array of another rank or dtype) raises TypeError, a value outside its domain\n" + "ValueError. Integers are read without narrowing, so the error does not depend on how the\n" + "input is represented.") + .def("uid", &Graph::uid, "process-unique id (never reused)") + .def("resolution", &Graph::resolution, "the resolution q (units of capacity)") + .def("num_nodes", &Graph::num_nodes, "N: node ids are 0 .. N - 1") + .def("num_edges", &Graph::num_edges, "E: EdgeIds are 0 .. E - 1") + .def("max_edge_cost", &Graph::max_edge_cost, "largest edge cost (0 without edges)") + .def("edge_cost_gcd", &Graph::edge_cost_gcd, "gcd of the edge costs (0 without edges)") + .def( + "edge_capacity_units", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.edge_capacity_units(), self); + }, + "int64[E] capacities in units, id order (read-only view)") + .def( + "edge_costs", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.edge_costs(), self); + }, + "int64[E] costs, id order (read-only view)") + .def( + "edge_sources", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.edge_sources(), self); + }, + "int32[E] source node per edge (read-only view)") + .def( + "edge_destinations", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.edge_destinations(), self); + }, + "int32[E] destination node per edge (read-only view)") + .def( + "input_positions", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.input_positions(), self); + }, + "int32[E]: EdgeId -> input position (read-only view); build the inverse yourself if needed") + .def( + "outgoing_offsets", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.outgoing_offsets(), self); + }, + "int32[N + 1]: the out-edges of u are the ids [outgoing_offsets()[u], outgoing_offsets()[u + 1])") + .def( + "incoming_offsets", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.incoming_offsets(), self); + }, + "int32[N + 1]: the in-edges of v are incoming_edges()[incoming_offsets()[v] : incoming_offsets()[v + 1]]") + .def( + "incoming_edges", [](const py::object& self) { + const auto& g = self.cast(); + return readonly_view(g.incoming_edges(), self); + }, + "int32[E]: the in-edges of v in (cost, source, input) order, as EdgeIds") + .def( + "eligibility", + [](const Graph& g, const py::object& excluded_nodes, const py::object& excluded_edges) { + // Nodes, then edges, each range-checked before narrowing (the constructor's order). + const auto xn = wide_integers(excluded_nodes, "excluded_nodes"); + for (std::size_t i = 0; i < xn.size(); ++i) { + if (!xn.in(i, 0, static_cast(g.num_nodes()) - 1)) { + throw py::value_error("excluded node " + xn.text(i) + " out of node range [0, " + + std::to_string(g.num_nodes()) + ")"); + } + } + const auto xe = wide_integers(excluded_edges, "excluded_edges"); + for (std::size_t i = 0; i < xe.size(); ++i) { + if (!xe.in(i, 0, static_cast(g.num_edges()) - 1)) { + throw py::value_error("excluded edge " + xe.text(i) + " out of edge range [0, " + + std::to_string(g.num_edges()) + ")"); + } + } + const auto nodes = xn.narrow(); + const auto edges = xe.narrow(); + return native_call([&] { return std::make_shared(g.eligibility(nodes, edges)); }); + }, + py::arg("excluded_nodes") = py::tuple(), py::arg("excluded_edges") = py::tuple(), + "Static eligibility from excluded node ids and EdgeIds (duplicates and unsorted lists\n" + "accepted, out-of-range ids rejected); an edge is allowed when it is not excluded and\n" + "both endpoints are allowed. Empty lists mean all allowed (uid 0)."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/hooks.cpp b/bindings/python/hooks.cpp new file mode 100644 index 0000000..3b37734 --- /dev/null +++ b/bindings/python/hooks.cpp @@ -0,0 +1,315 @@ +// The private hooks (NETGRAPH_CORE_PRIVATE_HOOKS builds only): the test controls of Python and +// native allocation failures and of lowered forwarding size limits, and the apportionment kernels. +#include +#include +#include + +#include "registration.hpp" + +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/forwarding.hpp" +#include "netgraph/core/detail/route_kernels.hpp" +#include "netgraph/core/detail/private_hooks.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/types.hpp" +#include "integers.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// Take the normalized exception, including its traceback, without replacing an existing +// exception instance. PyErr_GetRaisedException was added in Python 3.12. +py::object take_raised_exception() { +#if PY_VERSION_HEX >= 0x030C0000 + return py::reinterpret_steal(PyErr_GetRaisedException()); +#else + PyObject* type = nullptr; + PyObject* value = nullptr; + PyObject* traceback = nullptr; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + const auto kind = py::reinterpret_steal(type); + auto error = py::reinterpret_steal(value); + const auto trace = py::reinterpret_steal(traceback); + if (trace && PyException_SetTraceback(error.ptr(), trace.ptr()) < 0) throw py::error_already_set(); + return error; +#endif +} + +// The allocation-failure hook: the k-th object-domain allocation fails (Python's allocator +// wrapped for one call, then restored). The allocator is process-wide while the call runs, so +// the hook serves one caller at a time (the tests); its counters are atomics because the module +// does not hold the GIL for every thread (mod_gil_not_used). +PyMemAllocatorEx g_saved_allocator {}; +std::atomic g_fail_countdown {-1}; +std::atomic g_fail_seen {0}; +std::atomic g_fail_hit {false}; +std::atomic g_numpy_only {false}; + +bool fail_now() { + if (g_numpy_only.load() && !in_numpy_conversion) return false; + g_fail_seen.fetch_add(1); + if (g_fail_countdown.load() < 0) return false; + if (g_fail_countdown.fetch_sub(1) == 0) { + g_fail_hit.store(true); + return true; + } + return false; +} +void* failing_malloc(void*, std::size_t n) { return fail_now() ? nullptr : g_saved_allocator.malloc(g_saved_allocator.ctx, n); } +void* failing_calloc(void*, std::size_t n, std::size_t size) { + return fail_now() ? nullptr : g_saved_allocator.calloc(g_saved_allocator.ctx, n, size); +} +void* failing_realloc(void*, void* ptr, std::size_t n) { + return fail_now() ? nullptr : g_saved_allocator.realloc(g_saved_allocator.ctx, ptr, n); +} +void forwarding_free(void*, void* ptr) { g_saved_allocator.free(g_saved_allocator.ctx, ptr); } + +// The calling thread's trace function (a coverage tool's tracer) suspended while the guard lives: +// a tracer allocates at the calls and lines of the swept function and absorbs or translates its +// own failures, which are not the function's. +class Untraced { + public: + Untraced() + : trace_(PyThreadState_Get()->c_tracefunc), + arg_(py::reinterpret_borrow(PyThreadState_Get()->c_traceobj)) { + PyEval_SetTrace(nullptr, nullptr); + } + ~Untraced() { PyEval_SetTrace(trace_, arg_.ptr()); } + Untraced(const Untraced&) = delete; + Untraced& operator=(const Untraced&) = delete; + + private: + Py_tracefunc trace_; + py::object arg_; +}; + +// The native allocation-failure hook: while armed for one call, the k-th allocation of +// the calling thread through the extension's global operator new (the bindings and the library +// linked into the extension) throws std::bad_alloc, and the thread's allocations and deletions +// are counted, so a test also sees what a failed call left allocated. Other threads and other +// images (libc++, Python) are not counted; the aligned forms are not replaced. The counters are +// atomics for the same reason as above. +std::atomic g_native_armed {false}; +std::atomic g_native_owner {}; +std::atomic g_native_countdown {-1}; +std::atomic g_native_seen {0}; +std::atomic g_native_live {0}; +std::atomic g_native_hit {false}; + +bool native_counted() noexcept { + return g_native_armed.load(std::memory_order_relaxed) && g_native_owner.load() == std::this_thread::get_id(); +} + +// Registration may lazily allocate a bucket table which pybind11 keeps after +// the last instance is removed. In a free-threaded build the address of a new +// result can select a previously unused shard despite warming the operation. +// Prepare only those interpreter-owned empty tables before counting the call; +// native buffers, holders and registry nodes still fail/count in the call. +void prepare_native_registry() { + auto prepare = [](py::detail::instance_map& instances) { + if (instances.empty() && instances.bucket_count() <= 1) instances.reserve(1); + }; + auto& internals = py::detail::get_internals(); +#ifdef Py_GIL_DISABLED + for (std::size_t i = 0; i <= internals.instance_shards_mask; ++i) { + auto& shard = internals.instance_shards[i]; + std::unique_lock lock(shard.mutex); + prepare(shard.registered_instances); + } +#else + prepare(internals.registered_instances); +#endif +} + +void* native_allocate(std::size_t n) { + if (native_counted()) { + g_native_seen.fetch_add(1); + if (g_native_countdown.load() >= 0 && g_native_countdown.fetch_sub(1) == 0) { + g_native_hit.store(true); + throw std::bad_alloc(); + } + } + void* p = std::malloc(n == 0 ? 1 : n); + if (p == nullptr) throw std::bad_alloc(); + if (native_counted()) g_native_live.fetch_add(1); + return p; +} + +void* native_allocate_nothrow(std::size_t n) noexcept { + try { + return native_allocate(n); + } catch (...) { + return nullptr; + } +} + +void native_free(void* p) noexcept { + if (p != nullptr && native_counted()) g_native_live.fetch_sub(1); + std::free(p); +} + +} // namespace + +void register_private_hooks(py::module_& m) { + register_concurrency_hooks(m); + m.def( + "_fail_python_allocation", + [](const py::object& fn, const py::tuple& args, long k, bool numpy_only) { + // Calls fn(*args) with the k-th Python object allocation (from 0) failing, then + // restores the allocator. Returns (hit, allocations seen, result or None, exception or + // None). One caller at a time: the allocator is process-wide while the call runs. + const Untraced untraced; + g_numpy_only.store(numpy_only); + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &g_saved_allocator); + PyMemAllocatorEx hook {nullptr, failing_malloc, failing_calloc, failing_realloc, forwarding_free}; + g_fail_countdown.store(k); + g_fail_seen.store(0); + g_fail_hit.store(false); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &hook); + PyObject* r = PyObject_CallObject(fn.ptr(), args.ptr()); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &g_saved_allocator); + g_fail_countdown.store(-1); + g_numpy_only.store(false); + py::object result = r != nullptr ? py::reinterpret_steal(r) : py::object(py::none()); + py::object error = py::none(); + if (r == nullptr) error = take_raised_exception(); + return py::make_tuple(g_fail_hit.load(), g_fail_seen.load(), result, error); + }, + py::arg("fn"), py::arg("args"), py::arg("k"), py::arg("numpy_only") = false, + "private test hook: fn(*args) with the k-th Python object allocation failing -> (hit, seen, result, error);\n" + "numpy_only=True counts and fails only allocations inside NumPy casts, excluding other binding allocations"); + + m.def( + "_fail_native_allocation", + [](const py::object& fn, const py::tuple& args, long k) { + // Calls fn(*args) with the calling thread's k-th native allocation (from 0) failing. + // Returns (hit, allocations seen, result or None, exception or None, allocations of the + // call still live). One caller at a time. + prepare_native_registry(); + g_native_countdown.store(k); + g_native_seen.store(0); + g_native_live.store(0); + g_native_hit.store(false); + g_native_owner.store(std::this_thread::get_id()); + g_native_armed.store(true); + PyObject* r = PyObject_CallObject(fn.ptr(), args.ptr()); + g_native_armed.store(false); + g_native_countdown.store(-1); + py::object result = r != nullptr ? py::reinterpret_steal(r) : py::object(py::none()); + py::object error = py::none(); + if (r == nullptr) error = take_raised_exception(); + return py::make_tuple(g_native_hit.load(), g_native_seen.load(), result, error, g_native_live.load()); + }, + py::arg("fn"), py::arg("args"), py::arg("k"), + "private test hook: fn(*args) with the k-th native allocation failing -> (hit, seen, result, error, live)"); + + m.def( + "_with_forwarding_limits", + [](const py::object& fn, const py::tuple& args, const py::object& limits) { + // Calls fn(*args) with this thread's forwarding size limits lowered (entry, occurrence, + // router, event, dependency; 0 keeps the contract's limit), then restores them. Returns + // (result or None, exception or None, the event structures the call's last forwarding + // allocated). + const auto w = wide_integers(limits, "limits"); + if (w.size() != 5) throw py::value_error("limits needs 5 entries (got " + std::to_string(w.size()) + ")"); + std::array lowered {}; + for (std::size_t i = 0; i < 5; ++i) { + if (!w.in(i, 0, detail::kForwardingLimit)) throw py::value_error("limits[" + std::to_string(i) + "] = " + w.text(i) + " is out of range"); + lowered[i] = w.value[i]; + } + auto& current = detail::thread_forwarding_limits(); + const auto saved = current; + current = lowered; + detail::thread_lossy_counters().event_structures = 0; + PyObject* r = PyObject_CallObject(fn.ptr(), args.ptr()); + current = saved; + py::object result = r != nullptr ? py::reinterpret_steal(r) : py::object(py::none()); + py::object error = py::none(); + if (r == nullptr) error = take_raised_exception(); + return py::make_tuple(result, error, detail::thread_lossy_counters().event_structures); + }, + py::arg("fn"), py::arg("args"), py::arg("limits"), + "private test hook: fn(*args) with the five forwarding size limits lowered (0: unchanged) ->\n" + "(result, error, event structures)"); + + m.def( + "_check_forwarding_count", + [](const py::object& counter, const py::object& n) { + // The forwarding sizes' counting step on one exact count (counter: entry 0, occurrence 1, + // router 2, event 3, dependency 4), under this thread's limits. + const auto c = integer_value(counter, 0, 4, "counter"); + const auto v = integer_value(n, 0, std::numeric_limits::max(), "n"); + detail::check_forwarding_count(static_cast(c), v); + }, + py::arg("counter"), py::arg("n"), + "private: the forwarding size check of one count (entry, occurrence, router, event, dependency:\n" + "0 to 4); raises the limit's ValueError above it"); + + m.def( + "_round_robin", + [](const py::object& total, const py::object& k) { + const auto t = integer_value(total, 0, kDomainLimit - 1, "total"); + const auto n = integer_value(k, 1, 1 << 20, "k"); + std::vector out(static_cast(n)); + detail::round_robin(t, out); + return out; + }, + py::arg("total"), py::arg("k"), + "private: round robin of total units over k members (the first total % k get one unit more)"); + + m.def( + "_highest_averages", + [](const py::object& total, const py::object& weights) { + const auto t = integer_value(total, 0, kDomainLimit - 1, "total"); + const auto w = wide_integers(weights, "weights"); + if (w.size() == 0) throw py::value_error("highest averages over no member"); + for (std::size_t i = 0; i < w.size(); ++i) { + if (!w.in(i, 1, kDomainLimit - 1)) { + throw py::value_error("weights[" + std::to_string(i) + "] = " + w.text(i) + " is outside [1, 2^62)"); + } + } + const auto wv = w.narrow(); + std::vector out(wv.size()); + detail::highest_averages(t, wv, out); + return out; + }, + py::arg("total"), py::arg("weights"), + "private: highest-averages (D'Hondt) apportionment of total units over positive weights, ties by\n" + "the lower index"); +} + +} // namespace netgraph::core::bindings + +// The extension's global allocation functions in a hooks build: the native allocation-failure +// hook above. Unarmed they allocate as the defaults do (malloc and free, which every image's +// operator delete accepts). +using netgraph::core::bindings::native_allocate; +using netgraph::core::bindings::native_allocate_nothrow; +using netgraph::core::bindings::native_free; + +void* operator new(std::size_t n) { return native_allocate(n); } +void* operator new[](std::size_t n) { return native_allocate(n); } +void* operator new(std::size_t n, const std::nothrow_t&) noexcept { return native_allocate_nothrow(n); } +void* operator new[](std::size_t n, const std::nothrow_t&) noexcept { return native_allocate_nothrow(n); } +void operator delete(void* p) noexcept { native_free(p); } +void operator delete[](void* p) noexcept { native_free(p); } +void operator delete(void* p, std::size_t) noexcept { native_free(p); } +void operator delete[](void* p, std::size_t) noexcept { native_free(p); } +void operator delete(void* p, const std::nothrow_t&) noexcept { native_free(p); } +void operator delete[](void* p, const std::nothrow_t&) noexcept { native_free(p); } + +#endif diff --git a/bindings/python/integers.hpp b/bindings/python/integers.hpp new file mode 100644 index 0000000..eea1200 --- /dev/null +++ b/bindings/python/integers.hpp @@ -0,0 +1,253 @@ +/* + Integer arguments from Python: Python ints, NumPy integer scalars and integer arrays, read + without narrowing (CheckedIntegerVector) and validated in the C++ constructors' order before any + conversion to the C++ types, so the reported error does not depend on how the input is + represented; bools and floats are rejected, never truncated. Also the shape step of sequence + arguments, the per-edge amount arrays and the node sets. +*/ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/types.hpp" +#include "strict_bool.hpp" +#include "numpy_conversion.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// A Python bool or a numpy.bool_, by type identity (as StrictBool reads them). +inline bool is_bool(const py::handle& obj) { + return PyBool_Check(obj.ptr()) || reinterpret_cast(Py_TYPE(obj.ptr())) == numpy_bool_type(); +} + +// A wrong-type integer argument (a bool, a float, a string, None, another class): TypeError at +// every position, the value errors of a well-typed integer being ValueError (the library's one rule +// for the class of an input error). +[[noreturn]] inline void throw_not_an_integer(const py::handle& item, const std::string& what) { + throw py::type_error(what + " must be an integer (got " + std::string(py::str(py::type::of(item).attr("__name__"))) + ")"); +} + +// The Python int of any non-bool object with __index__, read with PyNumber_Index; bools and +// objects without that protocol are rejected (throw_not_an_integer), and an exception of the +// object's own __index__ propagates as raised. The result is an exact int, so PyLong_AsLongLongAndOverflow +// cannot fail on it. +inline py::object python_int(const py::handle& item, const std::string& what) { + if (is_bool(item) || !PyIndex_Check(item.ptr())) throw_not_an_integer(item, what); + py::object idx = py::reinterpret_steal(PyNumber_Index(item.ptr())); + if (!idx) throw py::error_already_set(); + return idx; +} + +// One integer of [lo, hi] (scalar arguments). +inline std::int64_t integer_value(const py::handle& item, std::int64_t lo, std::int64_t hi, const std::string& what) { + py::object idx = python_int(item, what); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < lo || v > hi) { + throw py::value_error(what + " = " + std::string(py::str(idx)) + " is out of range [" + + std::to_string(lo) + ", " + std::to_string(hi) + "]"); + } + return static_cast(v); +} + +// A 1-D integer sequence read without narrowing: every entry's int64 value where it fits, and +// the sign of the overflow where it does not (+1 above INT64_MAX, -1 below INT64_MIN). Only type +// defects raise here (a non-integer, a bool, a float array, a non-sequence); every value bound +// is checked by the caller in the C++ constructor's order, so the reported error does not depend +// on the representation of the input. +struct CheckedIntegerVector { + py::object source; + std::vector value; + std::vector overflow; // empty when no entry overflows + [[nodiscard]] std::size_t size() const noexcept { return value.size(); } + [[nodiscard]] int sign(std::size_t i) const noexcept { return overflow.empty() ? 0 : overflow[i]; } + // Entry i of the source as given, through a checked call (pybind11's int index reports a + // failed allocation as RuntimeError). + [[nodiscard]] py::object item(std::size_t i) const { + PyObject* o = PySequence_GetItem(source.ptr(), static_cast(i)); + if (o == nullptr) throw py::error_already_set(); + return py::reinterpret_steal(o); + } + // The entry as Python prints it (exact also beyond int64). + [[nodiscard]] std::string text(std::size_t i) const { + if (sign(i) == 0) return std::to_string(value[i]); + return std::string(py::str(item(i))); + } + [[nodiscard]] bool in(std::size_t i, std::int64_t lo, std::int64_t hi) const noexcept { + return sign(i) == 0 && value[i] >= lo && value[i] <= hi; + } + // The entry as an exact Python int (an overflowing entry compared with another). The entry is + // converted again here, so its __index__ may now fail: that exception propagates as raised. + [[nodiscard]] py::object exact(std::size_t i) const { + py::object entry = item(i); + py::object idx = py::reinterpret_steal(PyNumber_Index(entry.ptr())); + if (!idx) throw py::error_already_set(); + return idx; + } + template + [[nodiscard]] std::vector narrow() const { // after every bound was checked + return std::vector(value.begin(), value.end()); + } +}; + +inline bool is_invalid_sequence(const py::handle& obj) { + return py::isinstance(obj) || py::isinstance(obj) || !py::isinstance(obj); +} + +// A type or shape defect of a sequence argument (a non-sequence, a string, a nested entry, an +// array of another rank or of a non-integer dtype): TypeError, as for the scalars +// (throw_not_an_integer); its length is a value. +[[noreturn]] inline void throw_vector_defect(const std::string& text) { throw py::type_error(text); } + +inline CheckedIntegerVector wide_integers(const py::handle& obj, const char* name) { + CheckedIntegerVector out; + out.source = py::reinterpret_borrow(obj); + if (py::isinstance(obj)) { + auto arr = py::reinterpret_borrow(obj); + if (arr.ndim() != 1) { + throw_vector_defect(std::string(name) + " must be a 1-D array (got " + std::to_string(arr.ndim()) + " dimensions)"); + } + const char kind = arr.dtype().kind(); + if (kind != 'i' && kind != 'u') { + throw_vector_defect(std::string(name) + " entries must be integers (got dtype " + std::string(py::str(arr.dtype())) + ")"); + } + out.value.reserve(static_cast(arr.size())); + if (kind == 'u' && arr.itemsize() == 8) { + const auto u = numpy_conversion(arr); + auto r = u.unchecked<1>(); + constexpr auto kMax = static_cast(std::numeric_limits::max()); + for (py::ssize_t i = 0; i < r.shape(0); ++i) { + if (r(i) > kMax) { + if (out.overflow.empty()) out.overflow.assign(static_cast(r.shape(0)), 0); + out.overflow[static_cast(i)] = 1; + out.value.push_back(std::numeric_limits::max()); + } else { + out.value.push_back(static_cast(r(i))); + } + } + return out; + } + const auto a = numpy_conversion(arr); + auto r = a.unchecked<1>(); + for (py::ssize_t i = 0; i < r.shape(0); ++i) out.value.push_back(r(i)); + return out; + } + if (is_invalid_sequence(obj)) throw_vector_defect(std::string(name) + " must be a sequence of integers"); + auto seq = py::reinterpret_borrow(obj); + const std::size_t n = seq.size(); + out.value.reserve(n); + for (std::size_t i = 0; i < n; ++i) { + py::object idx = python_int(seq[i], std::string(name) + "[" + std::to_string(i) + "]"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0) { + if (out.overflow.empty()) out.overflow.assign(n, 0); + out.overflow[i] = static_cast(overflow); + out.value.push_back(overflow > 0 ? std::numeric_limits::max() + : std::numeric_limits::min()); + } else { + out.value.push_back(static_cast(v)); + } + } + return out; +} + +// An entry that is itself a sequence or an array of rank >= 1 (a shape defect of its argument). +inline bool is_nested(const py::handle& item) { + if (py::isinstance(item)) return py::reinterpret_borrow(item).ndim() >= 1; + return !py::isinstance(item) && !py::isinstance(item) && PySequence_Check(item.ptr()) != 0; +} + +// The shape step: the length of a rank-1 integer sequence argument. A NumPy array of another +// rank (0-D included), a nested sequence or a non-sequence is a shape defect. Entry types and +// values are not read here; their steps come later. +inline std::size_t sequence_length(const py::handle& obj, const char* name) { + if (py::isinstance(obj)) { + auto arr = py::reinterpret_borrow(obj); + if (arr.ndim() != 1) { + throw_vector_defect(std::string(name) + " must be a 1-D array (got " + std::to_string(arr.ndim()) + " dimensions)"); + } + return static_cast(arr.size()); + } + if (is_invalid_sequence(obj)) { + throw_vector_defect(std::string(name) + " must be a 1-D sequence of integers (got " + + std::string(py::str(py::type::of(obj).attr("__name__"))) + ")"); + } + auto seq = py::reinterpret_borrow(obj); + const std::size_t n = seq.size(); + for (std::size_t i = 0; i < n; ++i) { + if (is_nested(seq[i])) { + throw_vector_defect(std::string(name) + " must be a 1-D sequence of integers (entry " + std::to_string(i) + + " is a sequence)"); + } + } + return n; +} + +// A per-edge integer array in [0, the edge's capacity] (a ledger baseline, a search residual, +// the capacities `view` of _solve_min_cost_flow), validated on the unnarrowed values: the length +// (shape step), then every entry in EdgeId order. `label` names the entries in a type error +// ("residual (units)", "view (capacity units)"). The checked values come back unnarrowed: a +// caller narrows them later (_solve_min_cost_flow, inside its call). +inline CheckedIntegerVector edge_amounts(const py::object& obj, const Graph& graph, const char* name, const char* label) { + const auto num_edges = static_cast(graph.num_edges()); + const std::size_t len = sequence_length(obj, name); + if (len != num_edges) { + throw py::value_error(std::string(name) + " has length " + std::to_string(len) + ", expected " + + std::to_string(num_edges) + " (one entry per edge)"); + } + auto v = wide_integers(obj, label); + const auto capacity = graph.edge_capacity_units(); + for (std::size_t e = 0; e < num_edges; ++e) { + if (!v.in(e, 0, capacity[e])) { + throw py::value_error(std::string(name) + "[" + std::to_string(e) + "] = " + v.text(e) + " is outside [0, edge_capacity_units[" + + std::to_string(e) + "] = " + std::to_string(capacity[e]) + "]"); + } + } + return v; +} + +// A per-edge integer array in [0, the edge's capacity] (a ledger baseline, a search residual), +// validated on the unnarrowed values: the length (shape step), then every entry. +inline std::vector edge_units(const py::object& obj, const Graph& graph, const char* name) { + return edge_amounts(obj, graph, name, (std::string(name) + " (units)").c_str()).narrow(); +} + +// A node set's ids, non-empty and range-checked before narrowing (detail::node_set's messages). +inline std::vector checked_ids(const CheckedIntegerVector& ids, std::int32_t num_nodes, const char* name) { + if (ids.size() == 0) throw py::value_error(detail::empty_set_error(name)); + for (std::size_t i = 0; i < ids.size(); ++i) { + if (!ids.in(i, 0, static_cast(num_nodes) - 1)) { + throw py::value_error(detail::node_range_error(name, ids.text(i), num_nodes)); + } + } + return ids.narrow(); +} + +// A node set: one NodeId or a sequence of them. +inline std::vector node_ids(const py::object& obj, std::int32_t num_nodes, const char* name) { + if (!is_bool(obj) && PyIndex_Check(obj.ptr()) && !py::isinstance(obj)) { + // A one-entry list made through a checked allocation (pybind11's list constructor reports a + // failed allocation as RuntimeError). + PyObject* raw = PyList_New(1); + if (raw == nullptr) throw py::error_already_set(); + auto one = py::reinterpret_steal(raw); + PyList_SET_ITEM(raw, 0, obj.inc_ref().ptr()); + return checked_ids(wide_integers(one, name), num_nodes, name); + } + return checked_ids(wide_integers(obj, name), num_nodes, name); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/module.cpp b/bindings/python/module.cpp index e01488f..beffe44 100644 --- a/bindings/python/module.cpp +++ b/bindings/python/module.cpp @@ -1,686 +1,75 @@ /* - Pybind11 module exposing NetGraph-Core C++ APIs to Python. - - Uses NumPy arrays with zero-copy spans where possible for FlowState/FlowGraph views (read-only) - - Returns distances as float64 arrays by default (inf for unreachable); dtype='int64' available - - Validates edge/node masks for bool dtype and length - - StrictMultiDiGraph view methods return copies (by design) to avoid exposing lifetime + Pybind11 module `_netgraph_core`: the graph on integer units with Eligibility and the unit + conversions, the route structures (PredecessorDAG, Path), the route choice and placement axes, the + ledger with its records, the forwarding programs, the policy engine with its configuration + and results, several policies' actions in one operation (apply_actions), the ShortestPathCache, + the evaluators (evaluate_demand, evaluate_pairs, evaluate_edge_failures) and the public search (spf, ksp, + resolve_to_paths); and one private entry: the flow core's _solve_min_cost_flow. + A build with the private hooks (CMake NETGRAPH_CORE_PRIVATE_HOOKS: the editable development + build) adds the test controls and the parity diagnostics. + + - Graph accessors return read-only NumPy views that keep the graph alive (no per-call copy). + - Integer inputs (capacities, costs, ids, views, caps) take any non-bool object with __index__, including Python ints and NumPy + integer scalars, and sequences or integer arrays; floats and bools are rejected with TypeError, never + truncated. Values are read without narrowing and validated in the C++ + constructors' order before any conversion to the C++ types. Physical values + enter only through to_units; the resolution is validated before anything else. + - One rule for the class of an input error: a type the position never takes TypeError, a value + outside its domain ValueError. std::invalid_argument from Core maps to ValueError; enumeration + domains are checked when a configuration or options value is consumed (ValueError). + - _solve_min_cost_flow is a private entry point for the flow checks: it returns the flow + core's payload as a dict of Python ints and lists. + - Amounts and counters cross as Python ints (exact, also beyond 64 bits); per-edge arrays as + int64 NumPy arrays in EdgeId order; +inf is accepted only as a request (float('inf')). + - Python spells constants in UPPER_SNAKE_CASE: UNREACHABLE is the C++ kUnreachable. */ #include -#include -#include -#include -#include -#include -#include "netgraph/core/k_shortest_paths.hpp" -#include "netgraph/core/max_flow.hpp" -#include "netgraph/core/flow_state.hpp" -#include "netgraph/core/shortest_paths.hpp" -#include "netgraph/core/strict_multidigraph.hpp" -#include "netgraph/core/types.hpp" -#include "netgraph/core/flow_graph.hpp" -#include "netgraph/core/flow.hpp" -#include "netgraph/core/flow_policy.hpp" -#include "netgraph/core/backend.hpp" -#include "netgraph/core/algorithms.hpp" -#include "netgraph/core/options.hpp" -#include "netgraph/core/profiling.hpp" +#include "registration.hpp" namespace py = pybind11; using namespace netgraph::core; -// Helper opaque wrappers (defined at file scope to ensure stable type identity) -struct PyBackend { BackendPtr impl; }; -struct PyGraph { GraphHandle handle; py::object keep_alive; std::int32_t num_nodes; std::int32_t num_edges; }; - -// Helpers to check NumPy arrays -namespace { -// Small helper: copy a span into a new NumPy array of the same dtype. -template -py::array_t copy_to_numpy(std::span s) { - py::array_t arr(s.size()); - if (!s.empty()) std::memcpy(arr.mutable_data(), s.data(), s.size()*sizeof(T)); - return arr; -} - -// Owns a bool[] buffer and exposes a const span view onto it. -struct BoolSpan { - std::unique_ptr buf; - std::span view; -}; - -// Validate 1-D C-contig NumPy bool array and deep-copy into bool[]. -static BoolSpan to_bool_span_from_numpy(py::object obj, std::size_t expected_len, const char* what) { - BoolSpan out; - if (obj.is_none()) return out; // empty span - py::array arr = obj.cast(); - if (!(arr.flags() & py::array::c_style)) { - throw py::type_error(std::string(what) + " must be C-contiguous"); - } - auto b = arr.request(); - if (b.ndim != 1 || b.format != py::format_descriptor::format()) { - throw py::type_error(std::string(what) + " must be 1-D bool"); - } - if (static_cast(b.shape[0]) != expected_len) { - throw py::type_error(std::string(what) + " length must equal " + std::to_string(expected_len)); - } - out.buf.reset(new bool[expected_len]); - // NumPy bool_ uses 1 byte; C++ bool is 1 byte on supported platforms. - std::memcpy(out.buf.get(), b.ptr, expected_len * sizeof(bool)); - out.view = std::span(out.buf.get(), expected_len); - return out; -} -} // namespace - -// Helpers to check NumPy arrays -template -static std::span as_span(const py::array& arr, const char* name) { - if (!py::isinstance>(arr)) { - throw py::type_error(std::string(name) + ": expected numpy array of correct dtype"); - } - if (!(arr.flags() & py::array::c_style)) { - throw py::type_error(std::string(name) + ": array must be C-contiguous (use np.ascontiguousarray)"); - } - auto buf = arr.request(); - if (buf.ndim != 1) { - throw py::type_error(std::string(name) + ": expected 1-D array"); - } - return std::span(static_cast(buf.ptr), static_cast(buf.size)); -} - PYBIND11_MODULE(_netgraph_core, m, py::mod_gil_not_used()) { - m.doc() = "NetGraph-Core C++ bindings"; - - py::enum_(m, "EdgeTieBreak") - .value("DETERMINISTIC", EdgeTieBreak::Deterministic) - .value("PREFER_HIGHER_RESIDUAL", EdgeTieBreak::PreferHigherResidual); - - py::class_(m, "EdgeSelection") - .def(py::init<>()) - .def(py::init([](bool multi_edge, bool require_capacity, EdgeTieBreak tie_break){ - EdgeSelection s; s.multi_edge = multi_edge; s.require_capacity = require_capacity; s.tie_break = tie_break; return s; - }), - py::kw_only(), - py::arg("multi_edge") = true, - py::arg("require_capacity") = false, - py::arg("tie_break") = EdgeTieBreak::Deterministic) - .def_readwrite("multi_edge", &EdgeSelection::multi_edge) - .def_readwrite("require_capacity", &EdgeSelection::require_capacity) - .def_readwrite("tie_break", &EdgeSelection::tie_break); - - py::enum_(m, "FlowPlacement") - .value("PROPORTIONAL", FlowPlacement::Proportional) - .value("EQUAL_BALANCED", FlowPlacement::EqualBalanced) - .value("EQUAL_BALANCED_LOSSY", FlowPlacement::EqualBalancedLossy); - - py::class_(m, "StrictMultiDiGraph") - .def_static( - "from_arrays", - [](std::int32_t num_nodes, - py::array src, py::array dst, - py::array capacity, py::array cost, - py::array ext_edge_ids) { - // public API: src/dst are int32; pass through as int32 to core - auto src_s = as_span(src, "src"); - auto dst_s = as_span(dst, "dst"); - if (src_s.size() != dst_s.size()) throw py::type_error("src and dst must have the same length"); - auto cap_s = as_span(capacity, "capacity"); - // Cost dtype must be int64 to match internal Cost type - auto cost_s = as_span(cost, "cost"); - // ext_edge_ids required: maps internal edge indices to caller's stable IDs - auto ext_s = as_span(ext_edge_ids, "ext_edge_ids"); - return StrictMultiDiGraph::from_arrays(num_nodes, - src_s, - dst_s, - cap_s, cost_s, ext_s); - }, - py::arg("num_nodes"), py::arg("src"), py::arg("dst"), py::arg("capacity"), py::arg("cost"), - py::arg("ext_edge_ids")) - .def("num_nodes", &StrictMultiDiGraph::num_nodes) - .def("num_edges", &StrictMultiDiGraph::num_edges) - .def("capacity_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.capacity_view()); }) - .def("edge_src_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.edge_src_view()); }) - .def("edge_dst_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.edge_dst_view()); }) - .def("ext_edge_ids_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.ext_edge_ids_view()); }) - .def("cost_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.cost_view()); }) - .def("row_offsets_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.row_offsets_view()); }) - .def("col_indices_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.col_indices_view()); }) - .def("adj_edge_index_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.adj_edge_index_view()); }) - .def("in_row_offsets_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.in_row_offsets_view()); }) - .def("in_col_indices_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.in_col_indices_view()); }) - .def("in_adj_edge_index_view", [](const StrictMultiDiGraph& g){ return copy_to_numpy(g.in_adj_edge_index_view()); }); - - // Backend and Algorithms - - // Opaque wrapper types - py::class_(m, "Backend") - .def_static("cpu", [](){ return PyBackend{ make_cpu_backend() }; }); - - py::class_(m, "Graph") - // Opaque holder; constructed via Algorithms.build_graph - .def_property_readonly("num_nodes", [](const PyGraph& pg){ return pg.num_nodes; }) - .def_property_readonly("num_edges", [](const PyGraph& pg){ return pg.num_edges; }); - - py::class_>(m, "Algorithms") - .def(py::init([](const PyBackend& b){ return std::make_shared(b.impl); }), py::arg("backend")) - .def("build_graph", [](const Algorithms& algs, py::object graph_obj){ - // graph_obj is py::object so PyGraph can keep it alive; check the type - // explicitly, since a bare .cast<> failure surfaces as an opaque - // "RuntimeError: Unable to cast ... to C++ type '?'". - if (!py::isinstance(graph_obj)) { - throw py::type_error("build_graph: graph must be a StrictMultiDiGraph"); - } - const StrictMultiDiGraph& g = graph_obj.cast(); - auto gh = algs.build_graph(g); - // Keep the Python graph object alive alongside the handle when - // referencing a non-owned graph instance. - return PyGraph{ gh, graph_obj, g.num_nodes(), g.num_edges() }; - }, py::arg("graph")) - .def("spf", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, - py::object dst, py::object selection_obj, py::object residual_obj, - py::object node_mask, py::object edge_mask, bool multipath, std::string dtype) -> py::tuple { - if (src < 0 || src >= pg.num_nodes) throw py::value_error("src out of range"); - SpfOptions opts; if (!selection_obj.is_none()) opts.selection = py::cast(selection_obj); - if (!dst.is_none()) { auto d = py::cast(dst); if (d < 0 || d >= pg.num_nodes) throw py::value_error("dst out of range"); opts.dst = static_cast(d); } - std::vector residual_vec; - if (!residual_obj.is_none()) { - auto arr = py::cast(residual_obj); - if (!(arr.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous (np.ascontiguousarray)"); - auto buf = arr.request(); - if (buf.ndim != 1 || buf.format != py::format_descriptor::format()) throw py::type_error("residual must be 1-D float64"); - // Check length here so this matches the TypeError raised by every other - // length check (masks, FlowState residual); otherwise the core throws - // std::invalid_argument, which reaches Python as ValueError. - if (static_cast(buf.shape[0]) != pg.num_edges) { - throw py::type_error("residual length must equal " + std::to_string(pg.num_edges)); - } - residual_vec.resize(static_cast(buf.shape[0])); - std::memcpy(residual_vec.data(), buf.ptr, residual_vec.size()*sizeof(double)); - opts.residual = std::span(residual_vec.data(), residual_vec.size()); - } - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - opts.node_mask = node_bs.view; - opts.edge_mask = edge_bs.view; - opts.multipath = multipath; - py::gil_scoped_release rel; auto res = algs.spf(pg.handle, src, opts); py::gil_scoped_acquire acq; - auto maxc = std::numeric_limits::max(); - if (dtype == "int64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i(maxc) : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else if (dtype == "float64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i::infinity() : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else { - throw py::value_error("dtype must be 'float64' or 'int64'"); - } - }, py::arg("graph"), py::arg("src"), py::arg("dst") = py::none(), py::kw_only(), py::arg("selection") = py::none(), py::arg("residual") = py::none(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none(), py::arg("multipath") = true, py::arg("dtype") = "float64") - .def("spf_to", [](const Algorithms& algs, const PyGraph& pg, std::int32_t dst, - py::object selection_obj, py::object residual_obj, - py::object node_mask, py::object edge_mask, bool multipath, - py::object fanout_obj, std::string dtype) -> py::tuple { - if (dst < 0 || dst >= pg.num_nodes) throw py::value_error("dst out of range"); - SpfToOptions opts; if (!selection_obj.is_none()) opts.selection = py::cast(selection_obj); - std::vector residual_vec; - if (!residual_obj.is_none()) { - auto arr = py::cast(residual_obj); - if (!(arr.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous (np.ascontiguousarray)"); - auto buf = arr.request(); - if (buf.ndim != 1 || buf.format != py::format_descriptor::format()) throw py::type_error("residual must be 1-D float64"); - if (static_cast(buf.shape[0]) != pg.num_edges) { - throw py::type_error("residual length must equal " + std::to_string(pg.num_edges)); - } - residual_vec.resize(static_cast(buf.shape[0])); - std::memcpy(residual_vec.data(), buf.ptr, residual_vec.size()*sizeof(double)); - opts.residual = std::span(residual_vec.data(), residual_vec.size()); - } - std::vector fanout_vec; - if (!fanout_obj.is_none()) { - for (auto item : py::iterable(fanout_obj)) { - auto e = py::cast(item); - if (e < 0 || e >= pg.num_edges) throw py::value_error("fanout edge id out of range"); - fanout_vec.push_back(static_cast(e)); - } - opts.fanout_edges = std::span(fanout_vec.data(), fanout_vec.size()); - } - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - opts.node_mask = node_bs.view; - opts.edge_mask = edge_bs.view; - opts.multipath = multipath; - py::gil_scoped_release rel; auto res = algs.spf_to(pg.handle, dst, opts); py::gil_scoped_acquire acq; - auto maxc = std::numeric_limits::max(); - if (dtype == "int64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i(maxc) : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else if (dtype == "float64") { - py::array_t dist_arr(res.first.size()); - auto* out = dist_arr.mutable_data(); - for (std::size_t i=0;i::infinity() : static_cast(res.first[i]); - return py::make_tuple(std::move(dist_arr), std::move(res.second)); - } else { - throw py::value_error("dtype must be 'float64' or 'int64'"); - } - }, py::arg("graph"), py::arg("dst"), py::kw_only(), py::arg("selection") = py::none(), py::arg("residual") = py::none(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none(), py::arg("multipath") = true, py::arg("fanout_edges") = py::none(), py::arg("dtype") = "float64", - "Shortest paths from every node to dst (reverse SPF). Returns (distances_to_dst, dag); the DAG is forward-oriented and valid for placement from any node toward dst. fanout_edges are added to the DAG regardless of cost (see shortest_paths.hpp).") - .def("ksp", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, std::int32_t dst, - int k, py::object max_cost_factor, bool unique, py::object node_mask, py::object edge_mask, std::string dtype){ - if (src < 0 || src >= pg.num_nodes || dst < 0 || dst >= pg.num_nodes) throw py::value_error("src/dst out of range"); - if (k <= 0) throw py::value_error("k must be >= 1"); - if (dtype != "float64" && dtype != "int64") throw py::value_error("dtype must be 'float64' or 'int64'"); - KspOptions opts; opts.k = k; opts.unique = unique; if (!max_cost_factor.is_none()) opts.max_cost_factor = py::cast(max_cost_factor); - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - opts.node_mask = node_bs.view; opts.edge_mask = edge_bs.view; - py::gil_scoped_release rel; auto items = algs.ksp(pg.handle, src, dst, opts); py::gil_scoped_acquire acq; - py::list out; - auto maxc = std::numeric_limits::max(); - for (auto& pr : items) { - auto& dist = pr.first; - if (dtype == "int64") { - py::array_t dist_arr(dist.size()); - auto* outp = dist_arr.mutable_data(); - for (std::size_t i=0;i(maxc) : static_cast(dist[i]); - out.append(py::make_tuple(std::move(dist_arr), pr.second)); - } else if (dtype == "float64") { - py::array_t dist_arr(dist.size()); - auto* outp = dist_arr.mutable_data(); - for (std::size_t i=0;i::infinity() : static_cast(dist[i]); - out.append(py::make_tuple(std::move(dist_arr), pr.second)); - } - } - return out; - }, py::arg("graph"), py::arg("src"), py::arg("dst"), py::kw_only(), py::arg("k"), py::arg("max_cost_factor") = py::none(), py::arg("unique") = true, py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none(), py::arg("dtype") = "float64") - .def("max_flow", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, std::int32_t dst, - FlowPlacement placement, bool shortest_path, bool require_capacity, bool with_edge_flows, bool with_reachable, bool with_residuals, - py::object node_mask, py::object edge_mask){ - if (src < 0 || src >= pg.num_nodes || dst < 0 || dst >= pg.num_nodes) throw py::value_error("src/dst out of range"); - MaxFlowOptions o; o.placement = placement; o.shortest_path = shortest_path; o.require_capacity = require_capacity; o.with_edge_flows = with_edge_flows; o.with_reachable = with_reachable; o.with_residuals = with_residuals; - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - o.node_mask = node_bs.view; o.edge_mask = edge_bs.view; - py::gil_scoped_release rel; auto res = algs.max_flow(pg.handle, src, dst, o); py::gil_scoped_acquire acq; return res; - }, py::arg("graph"), py::arg("src"), py::arg("dst"), py::kw_only(), py::arg("flow_placement") = FlowPlacement::Proportional, py::arg("shortest_path") = false, py::arg("require_capacity") = true, py::arg("with_edge_flows") = false, py::arg("with_reachable") = false, py::arg("with_residuals") = false, py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()) - .def("batch_max_flow", [](const Algorithms& algs, const PyGraph& pg, py::array pairs, - py::object node_masks, py::object edge_masks, - FlowPlacement placement, bool shortest_path, bool require_capacity, - bool with_edge_flows, bool with_reachable, bool with_residuals){ - // Check the dtype, not the buffer format string: NumPy spells int32 as - // NPY_LONG ('l') on LLP64 (Windows) and NPY_INT ('i') on LP64, while - // format_descriptor is always 'i'. Comparing format strings - // therefore rejects a genuine int32 array on Windows. isinstance uses - // dtype equivalence, matching as_span() above. - if (!py::isinstance>(pairs)) throw py::type_error("pairs dtype must be int32"); - if (!(pairs.flags() & py::array::c_style)) throw py::type_error("pairs must be C-contiguous (use np.ascontiguousarray)"); - auto buf = pairs.request(); - if (buf.ndim != 2 || buf.shape[1] != 2) throw py::type_error("pairs must be shape [B,2]"); - const std::size_t B = static_cast(buf.shape[0]); - std::vector> pp; pp.reserve(B); - auto* p = static_cast(buf.ptr); - for (std::size_t i=0;i= pg.num_nodes || - pp[i].second < 0 || pp[i].second >= pg.num_nodes) { - throw py::value_error("pairs[" + std::to_string(i) + "] has a src/dst out of range"); - } - } - std::vector node_bufs, edge_bufs; - std::vector> node_spans; std::vector> edge_spans; - auto parse_mask_list = [&](py::object list_obj, std::size_t expected_len, const char* what, - std::vector& bufs, std::vector>& spans) { - if (list_obj.is_none()) return; - auto seq = py::cast(list_obj); - if (static_cast(py::len(seq)) != B) - throw py::type_error(std::string(what) + " length must equal number of pairs"); - bufs.reserve(B); spans.reserve(B); - for (auto item : seq) { - auto bs = to_bool_span_from_numpy(item, expected_len, what); - spans.push_back(bs.view); - bufs.push_back(std::move(bs)); // keep storage alive across GIL release - } - }; - parse_mask_list(node_masks, static_cast(pg.num_nodes), "node_masks", node_bufs, node_spans); - parse_mask_list(edge_masks, static_cast(pg.num_edges), "edge_masks", edge_bufs, edge_spans); - MaxFlowOptions o; o.placement = placement; o.shortest_path = shortest_path; o.require_capacity = require_capacity; o.with_edge_flows = with_edge_flows; o.with_reachable = with_reachable; o.with_residuals = with_residuals; - py::gil_scoped_release rel; auto out = algs.batch_max_flow(pg.handle, pp, o, node_spans, edge_spans); py::gil_scoped_acquire acq; return out; - }, py::arg("graph"), py::arg("pairs"), py::kw_only(), py::arg("node_masks") = py::none(), py::arg("edge_masks") = py::none(), py::arg("flow_placement") = FlowPlacement::Proportional, py::arg("shortest_path") = false, py::arg("require_capacity") = true, py::arg("with_edge_flows") = false, py::arg("with_reachable") = false, py::arg("with_residuals") = false) - .def("sensitivity_analysis", [](const Algorithms& algs, const PyGraph& pg, std::int32_t src, std::int32_t dst, - FlowPlacement placement, bool shortest_path, bool require_capacity, - py::object node_mask, py::object edge_mask){ - if (src < 0 || src >= pg.num_nodes || dst < 0 || dst >= pg.num_nodes) throw py::value_error("src/dst out of range"); - MaxFlowOptions o; o.placement = placement; o.shortest_path = shortest_path; o.require_capacity = require_capacity; - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - o.node_mask = node_bs.view; o.edge_mask = edge_bs.view; - py::gil_scoped_release rel; auto res = algs.sensitivity_analysis(pg.handle, src, dst, o); py::gil_scoped_acquire acq; - // Return as list of tuples (EdgeId, FlowDelta) - py::list out; - for (auto const& pr : res) { - out.append(py::make_tuple(pr.first, pr.second)); - } - return out; - }, py::arg("graph"), py::arg("src"), py::arg("dst"), py::kw_only(), py::arg("flow_placement") = FlowPlacement::Proportional, py::arg("shortest_path") = false, py::arg("require_capacity") = true, py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()); - - py::class_(m, "PredDAG") - .def_property_readonly("parent_offsets", [](const PredDAG& d){ - return copy_to_numpy(std::span(d.parent_offsets.data(), d.parent_offsets.size())); - }) - .def_property_readonly("parents", [](const PredDAG& d){ - return copy_to_numpy(std::span(d.parents.data(), d.parents.size())); - }) - .def_property_readonly("via_edges", [](const PredDAG& d){ - return copy_to_numpy(std::span(d.via_edges.data(), d.via_edges.size())); - }) - .def_static("from_edges", [](const StrictMultiDiGraph& g, py::sequence edges){ - std::vector e; - e.reserve(py::len(edges)); - for (auto item : edges) e.push_back(py::cast(item)); - return make_path_dag(g, e); - }, py::arg("graph"), py::arg("edges"), - "Build a single-path PredDAG from a contiguous edge-id sequence.\n" - "Validates that each edge exists, consecutive edges connect, and the walk " - "is a simple path. The result works anywhere a PredDAG is accepted " - "(place_on_dag, FlowGraph.place, FlowPolicy.set_static_paths).") - .def("resolve_to_paths", [](const PredDAG& dag, std::int32_t src, std::int32_t dst, bool split_parallel_edges, py::object max_paths){ - std::optional mp; - if (!max_paths.is_none()) mp = py::cast(max_paths); - auto out = netgraph::core::resolve_to_paths(dag, src, dst, split_parallel_edges, mp); - py::list paths; - for (auto& path : out) { - py::list py_elems; - for (auto& pr : path) { - py::tuple edge_tuple(pr.second.size()); - for (std::size_t i=0;i(m, "MinCut") - .def_property_readonly("edges", [](const MinCut& mc){ - return copy_to_numpy(std::span(mc.edges.data(), mc.edges.size())); - }); - - py::class_(m, "FlowSummary") - .def_readonly("total_flow", &FlowSummary::total_flow) - .def_readonly("min_cut", &FlowSummary::min_cut) - .def_property_readonly("costs", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.costs.data(), s.costs.size())); - }) - .def_property_readonly("flows", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.flows.data(), s.flows.size())); - }) - .def_property_readonly("edge_flows", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.edge_flows.data(), s.edge_flows.size())); - }) - .def_property_readonly("residual_capacity", [](const FlowSummary& s){ - return copy_to_numpy(std::span(s.residual_capacity.data(), s.residual_capacity.size())); - }) - .def_property_readonly("reachable_nodes", [](const FlowSummary& s){ - // Store as bool array for Python; cast bytes to bool - py::array_t arr(s.reachable_nodes.size()); - auto* out = arr.mutable_data(); - for (std::size_t i=0;i(s.reachable_nodes[i]); - return arr; - }); - - // FlowState bindings - py::class_(m, "FlowState") - .def(py::init([](const StrictMultiDiGraph& g){ return FlowState(g); }), - py::arg("graph"), py::keep_alive<1, 2>()) - .def(py::init([](const StrictMultiDiGraph& g, py::array residual){ - if (!(py::isinstance>(residual))) throw py::type_error("residual must be a numpy float64 array"); - if (!(residual.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous"); - auto buf = residual.request(); - if (buf.ndim != 1 || static_cast(buf.shape[0]) != static_cast(g.num_edges())) throw py::type_error("residual length must equal num_edges"); - std::span rspan(static_cast(buf.ptr), static_cast(buf.shape[0])); - return FlowState(g, rspan); - }), py::arg("graph"), py::arg("residual"), py::keep_alive<1, 2>()) - .def("reset", [](FlowState& fs){ fs.reset(); }) - .def("reset", [](FlowState& fs, py::array residual){ - if (!(py::isinstance>(residual))) throw py::type_error("residual must be a numpy float64 array"); - if (!(residual.flags() & py::array::c_style)) throw py::type_error("residual must be C-contiguous"); - auto buf = residual.request(); - if (buf.ndim != 1 || static_cast(buf.shape[0]) != static_cast(fs.capacity_view().size())) throw py::type_error("residual length must equal num_edges"); - std::span rspan(static_cast(buf.ptr), static_cast(buf.shape[0])); - fs.reset(rspan); - }) - .def("capacity_view", [](py::object self_obj){ - const FlowState& fs = py::cast(self_obj); - auto s = fs.capacity_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("residual_view", [](py::object self_obj){ - const FlowState& fs = py::cast(self_obj); - auto s = fs.residual_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("edge_flow_view", [](py::object self_obj){ - const FlowState& fs = py::cast(self_obj); - auto s = fs.edge_flow_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("place_on_dag", [](FlowState& fs, std::int32_t src, std::int32_t dst, const PredDAG& dag, double requested_flow, FlowPlacement placement){ - py::gil_scoped_release rel; auto placed = fs.place_on_dag(src, dst, dag, requested_flow, placement); py::gil_scoped_acquire acq; return placed; - }, py::arg("src"), py::arg("dst"), py::arg("dag"), py::arg("requested_flow") = std::numeric_limits::infinity(), py::arg("flow_placement") = FlowPlacement::Proportional) - .def("place_max_flow", [](py::object self_obj, std::int32_t src, std::int32_t dst, - FlowPlacement placement, bool shortest_path, bool require_capacity, - py::object node_mask, py::object edge_mask){ - FlowState& fs = py::cast(self_obj); - const StrictMultiDiGraph& g = fs.graph(); - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(g.num_nodes()), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(g.num_edges()), "edge_mask"); - py::gil_scoped_release rel; - auto total = fs.place_max_flow(src, dst, placement, shortest_path, require_capacity, - node_bs.view, edge_bs.view); - py::gil_scoped_acquire acq; - return total; - }, py::arg("src"), py::arg("dst"), - py::arg("flow_placement") = FlowPlacement::Proportional, - py::arg("shortest_path") = false, py::arg("require_capacity") = true, - py::kw_only(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()) - .def("compute_min_cut", [](py::object self_obj, std::int32_t src, py::object node_mask, py::object edge_mask){ - const FlowState& fs = py::cast(self_obj); - const StrictMultiDiGraph& g = fs.graph(); - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(g.num_nodes()), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(g.num_edges()), "edge_mask"); - py::gil_scoped_release rel; - auto mc = fs.compute_min_cut(src, node_bs.view, edge_bs.view); - py::gil_scoped_acquire acq; - return mc; - }, py::arg("src"), py::kw_only(), py::arg("node_mask") = py::none(), py::arg("edge_mask") = py::none()); - - // FlowIndex and FlowGraph bindings - py::class_(m, "FlowIndex") - .def(py::init()) - .def_readonly("src", &FlowIndex::src) - .def_readonly("dst", &FlowIndex::dst) - .def_readonly("flowClass", &FlowIndex::flowClass) - .def_readonly("flowId", &FlowIndex::flowId); - - py::class_(m, "FlowGraph") - .def(py::init([](const StrictMultiDiGraph& g){ return FlowGraph(g); }), - py::arg("graph"), py::keep_alive<1, 2>()) - .def("capacity_view", [](py::object self_obj){ - const FlowGraph& fg = py::cast(self_obj); - auto s = fg.capacity_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("residual_view", [](py::object self_obj){ - const FlowGraph& fg = py::cast(self_obj); - auto s = fg.residual_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def("edge_flow_view", [](py::object self_obj){ - const FlowGraph& fg = py::cast(self_obj); - auto s = fg.edge_flow_view(); - py::array view(py::buffer_info(const_cast(s.data()), sizeof(double), py::format_descriptor::format(), 1, { s.size() }, { sizeof(double) }), self_obj); - view.attr("setflags")(py::arg("write") = false); - return view; - }) - .def_property_readonly("graph", [](const FlowGraph& fg){ return &fg.graph(); }, py::return_value_policy::reference_internal) - .def("place", [](FlowGraph& fg, const FlowIndex& idx, std::int32_t src, std::int32_t dst, const PredDAG& dag, double amount, FlowPlacement placement){ py::gil_scoped_release rel; auto placed = fg.place(idx, src, dst, dag, amount, placement); py::gil_scoped_acquire acq; return placed; }, py::arg("index"), py::arg("src"), py::arg("dst"), py::arg("dag"), py::arg("amount"), py::arg("flow_placement") = FlowPlacement::Proportional) - .def("place_with_drops", [](FlowGraph& fg, const FlowIndex& idx, std::int32_t src, std::int32_t dst, const PredDAG& dag, double amount, FlowPlacement placement){ - std::vector> drops; - double placed; - { py::gil_scoped_release rel; placed = fg.place(idx, src, dst, dag, amount, placement, &drops); } - py::list out; for (auto const& pr : drops) out.append(py::make_tuple(pr.first, pr.second)); - return py::make_tuple(placed, out); - }, py::arg("index"), py::arg("src"), py::arg("dst"), py::arg("dag"), py::arg("amount"), py::arg("flow_placement") = FlowPlacement::EqualBalancedLossy, - "Like place(), but also returns the per-edge dropped volume as a list of (edge_id, dropped) pairs. Only EQUAL_BALANCED_LOSSY drops flow; other placements return an empty list.") - .def("remove", [](FlowGraph& fg, const FlowIndex& idx){ py::gil_scoped_release rel; fg.remove(idx); py::gil_scoped_acquire acq; }) - .def("remove_by_class", [](FlowGraph& fg, std::int32_t cls){ py::gil_scoped_release rel; fg.remove_by_class(cls); py::gil_scoped_acquire acq; }) - .def("reset", [](FlowGraph& fg){ py::gil_scoped_release rel; fg.reset(); py::gil_scoped_acquire acq; }) - .def("get_flow_edges", [](const FlowGraph& fg, const FlowIndex& idx){ auto v = fg.get_flow_edges(idx); py::list out; for (auto const& pr : v) { out.append(py::make_tuple(pr.first, pr.second)); } return out; }) - .def("get_flow_path", [](const FlowGraph& fg, const FlowIndex& idx){ auto v = fg.get_flow_path(idx); py::list out; for (auto e : v) out.append(e); return out; }); - - // NOTE: FlowRecord is deliberately not bound. No binding constructs or returns one - // (FlowPolicy.flows yields plain tuples), so the class was unreachable, and the - // Python name "Flow" collided with the C++ alias `using Flow = double`. If per-flow - // records become part of the public API, bind it under an unambiguous name such as - // "FlowRecord" and have FlowPolicy.flows return it instead of tuples. - - py::enum_(m, "PathAlg") - .value("SPF", PathAlg::SPF); - - py::class_(m, "FlowPolicyConfig") - .def(py::init<>()) - .def(py::init([](PathAlg path_alg, - FlowPlacement flow_placement, - EdgeSelection selection, - bool require_capacity, - bool multipath, - int min_flow_count, - py::object max_flow_count, - py::object max_path_cost, - py::object max_path_cost_factor, - bool shortest_path, - bool reoptimize_flows_on_each_placement, - int max_no_progress_iterations, - int max_total_iterations, - bool diminishing_returns_enabled, - int diminishing_returns_window, - double diminishing_returns_epsilon_frac){ - FlowPolicyConfig cfg; - cfg.path_alg = path_alg; - cfg.flow_placement = flow_placement; - cfg.selection = selection; - cfg.require_capacity = require_capacity; - cfg.multipath = multipath; - cfg.min_flow_count = min_flow_count; - if (!max_flow_count.is_none()) cfg.max_flow_count = py::cast(max_flow_count); - if (!max_path_cost.is_none()) cfg.max_path_cost = py::cast(max_path_cost); - if (!max_path_cost_factor.is_none()) cfg.max_path_cost_factor = py::cast(max_path_cost_factor); - cfg.shortest_path = shortest_path; - cfg.reoptimize_flows_on_each_placement = reoptimize_flows_on_each_placement; - cfg.max_no_progress_iterations = max_no_progress_iterations; - cfg.max_total_iterations = max_total_iterations; - cfg.diminishing_returns_enabled = diminishing_returns_enabled; - cfg.diminishing_returns_window = diminishing_returns_window; - cfg.diminishing_returns_epsilon_frac = diminishing_returns_epsilon_frac; - return cfg; - }), - py::kw_only(), - py::arg("path_alg") = PathAlg::SPF, - py::arg("flow_placement") = FlowPlacement::Proportional, - py::arg("selection") = EdgeSelection{}, - py::arg("require_capacity") = true, - py::arg("multipath") = true, - py::arg("min_flow_count") = 1, - py::arg("max_flow_count") = py::none(), - py::arg("max_path_cost") = py::none(), - py::arg("max_path_cost_factor") = py::none(), - py::arg("shortest_path") = false, - py::arg("reoptimize_flows_on_each_placement") = false, - py::arg("max_no_progress_iterations") = 100, - py::arg("max_total_iterations") = 10000, - py::arg("diminishing_returns_enabled") = true, - py::arg("diminishing_returns_window") = 8, - py::arg("diminishing_returns_epsilon_frac") = 1e-3) - .def_readwrite("path_alg", &FlowPolicyConfig::path_alg) - .def_readwrite("flow_placement", &FlowPolicyConfig::flow_placement) - .def_readwrite("selection", &FlowPolicyConfig::selection) - .def_readwrite("require_capacity", &FlowPolicyConfig::require_capacity) - .def_readwrite("multipath", &FlowPolicyConfig::multipath) - .def_readwrite("min_flow_count", &FlowPolicyConfig::min_flow_count) - .def_readwrite("max_flow_count", &FlowPolicyConfig::max_flow_count) - .def_readwrite("max_path_cost", &FlowPolicyConfig::max_path_cost) - .def_readwrite("max_path_cost_factor", &FlowPolicyConfig::max_path_cost_factor) - .def_readwrite("shortest_path", &FlowPolicyConfig::shortest_path) - .def_readwrite("reoptimize_flows_on_each_placement", &FlowPolicyConfig::reoptimize_flows_on_each_placement) - .def_readwrite("max_no_progress_iterations", &FlowPolicyConfig::max_no_progress_iterations) - .def_readwrite("max_total_iterations", &FlowPolicyConfig::max_total_iterations) - .def_readwrite("diminishing_returns_enabled", &FlowPolicyConfig::diminishing_returns_enabled) - .def_readwrite("diminishing_returns_window", &FlowPolicyConfig::diminishing_returns_window) - .def_readwrite("diminishing_returns_epsilon_frac", &FlowPolicyConfig::diminishing_returns_epsilon_frac); - - py::class_(m, "FlowPolicy") - .def("__init__", [](FlowPolicy& self, py::object algs_obj, const PyGraph& pg, FlowPolicyConfig cfg, - py::object node_mask, py::object edge_mask){ - if (!py::isinstance(algs_obj)) { - throw py::type_error("FlowPolicy: algorithms must be an Algorithms instance"); - } - std::shared_ptr algs = py::cast>(algs_obj); - - auto node_bs = to_bool_span_from_numpy(node_mask, static_cast(pg.num_nodes), "node_mask"); - auto edge_bs = to_bool_span_from_numpy(edge_mask, static_cast(pg.num_edges), "edge_mask"); - - cfg.node_mask = node_bs.view; - cfg.edge_mask = edge_bs.view; - - ExecutionContext ctx(algs, pg.handle); - new (&self) FlowPolicy(ctx, cfg); - }, - py::arg("algorithms"), py::arg("graph"), py::arg("config"), - py::kw_only(), - py::arg("node_mask") = py::none(), - py::arg("edge_mask") = py::none(), - py::keep_alive<1, 2>(), // self keeps algorithms alive - py::keep_alive<1, 3>()) // self keeps graph alive - .def("flow_count", &FlowPolicy::flow_count) - .def("placed_demand", &FlowPolicy::placed_demand) - .def("place_demand", [](FlowPolicy& p, FlowGraph& fg, std::int32_t src, std::int32_t dst, FlowClass flowClass, double volume, py::object target_per_flow, py::object min_flow){ std::optional tpf; if (!target_per_flow.is_none()) tpf = py::cast(target_per_flow); std::optional mfl; if (!min_flow.is_none()) mfl = py::cast(min_flow); py::gil_scoped_release rel; auto pr = p.place_demand(fg, src, dst, flowClass, volume, tpf, mfl); py::gil_scoped_acquire acq; return py::make_tuple(pr.first, pr.second); }, py::arg("flow_graph"), py::arg("src"), py::arg("dst"), py::arg("flowClass"), py::arg("volume"), py::arg("target_per_flow") = py::none(), py::arg("min_flow") = py::none()) - .def("rebalance_demand", [](FlowPolicy& p, FlowGraph& fg, std::int32_t src, std::int32_t dst, FlowClass flowClass, double target){ py::gil_scoped_release rel; auto pr = p.rebalance_demand(fg, src, dst, flowClass, target); py::gil_scoped_acquire acq; return py::make_tuple(pr.first, pr.second); }, - py::arg("flow_graph"), py::arg("src"), py::arg("dst"), py::arg("flowClass"), py::arg("target")) - .def("remove_demand", [](FlowPolicy& p, FlowGraph& fg){ py::gil_scoped_release rel; p.remove_demand(fg); py::gil_scoped_acquire acq; }) - .def("set_static_paths", [](FlowPolicy& p, std::int32_t src, std::int32_t dst, py::sequence paths){ - std::vector bundles; - bundles.reserve(py::len(paths)); - for (auto item : paths) bundles.push_back(py::cast(item)); - py::gil_scoped_release rel; - p.set_static_paths(src, dst, std::move(bundles)); - py::gil_scoped_acquire acq; - }, py::arg("src"), py::arg("dst"), py::arg("paths"), - "Pin this policy's demand to explicit path bundles (one flow per usable " - "bundle, bound in supply order). See FlowPolicy docs for validation, mask " - "(failure) semantics, and which config knobs become inert.") - .def_property_readonly("flows", [](const FlowPolicy& p){ py::dict out; for (auto const& kv : p.flows()) { const auto& idx = kv.first; const auto& f = kv.second; out[py::make_tuple(idx.src, idx.dst, idx.flowClass, idx.flowId)] = py::make_tuple(f.src, f.dst, f.cost, f.placed_flow); } return out; }); - - // Profiling functions (enabled via NGRAPH_CORE_PROFILE=1 environment variable) - m.def("profiling_enabled", &profiling_enabled, - "Return True if profiling is enabled (NGRAPH_CORE_PROFILE=1)."); - m.def("profiling_dump", []{ ProfilingStats::instance().dump(); }, - "Print profiling statistics to stderr."); - m.def("profiling_reset", []{ ProfilingStats::instance().reset(); }, - "Clear all profiling statistics."); + m.doc() = + "NetGraph-Core: the graph on integer units, the route structures, the integer ledger, the forwarding\n" + "programs, the policy engine FlowPolicy and its lists of actions (apply_actions), the ShortestPathCache,\n" + "the evaluators (evaluate_demand, evaluate_pairs, evaluate_edge_failures) and the searches (spf, ksp,\n" + "resolve_to_paths).\n" + "One rule for the class of an input error, at every public entry: a type the position never takes\n" + "(a bool at a numeric position, a float where only an integer is taken, a string, None where none\n" + "is allowed, another class; for a vector a non-sequence, a nested entry, an array of another rank\n" + "or of a non-integer dtype) raises TypeError, a value outside its domain (a range, a length, NaN,\n" + "-inf, a finite float where floats are taken only for +inf) ValueError; the text is the same.\n" + "Every public call either completes or raises and leaves Core state as it was, with two limits:\n" + "(1) pybind11 does not check every allocation when it constructs a bound object from Python, parses\n" + "the arguments of a call that passes keywords, or converts an attribute or a returned value into a\n" + "new Python object outside the library's checked result construction; running out of memory exactly\n" + "there may crash the process, with Core state unchanged. (2) A ShortestPathCache passed to the call: a building one keeps the complete entries and\n" + "statistics of the lookups that succeeded before the call failed; a frozen one counts one miss\n" + "when the call raised ShortestPathCacheMiss."; + + // In this order: a later binding's defaults and conversions need the earlier types. + bindings::register_options(m); + bindings::register_units(m); + bindings::register_eligibility(m); + bindings::register_graph(m); + bindings::register_flow_solver_hook(m); + bindings::register_predecessor_dag(m); + bindings::register_path(m); + bindings::register_flow_record(m); + bindings::register_flow_ledger(m); + bindings::register_program(m); + bindings::register_routing(m); + bindings::register_policy_config(m); + bindings::register_placement_result(m); + bindings::register_shortest_path_cache(m); + bindings::register_flow_policy(m); + bindings::register_apply_actions(m); + bindings::register_evaluators(m); + bindings::register_paths(m); + bindings::register_search(m); +#if NETGRAPH_CORE_PRIVATE_HOOKS + bindings::register_private_hooks(m); +#endif } diff --git a/bindings/python/native_call.hpp b/bindings/python/native_call.hpp new file mode 100644 index 0000000..1d03764 --- /dev/null +++ b/bindings/python/native_call.hpp @@ -0,0 +1,67 @@ +// The Python/native boundary. Only native values may cross this scope: all Python +// conversion and ownership work is done by the caller before or after it. +#pragma once + +#include +#include +#include +#include +#include +#include "netgraph/core/detail/concurrency.hpp" + +namespace netgraph::core::bindings { +// The GIL released for native work: every native_call, and a FlowPolicy's destructor (its holder's +// deleter, which waits for the ledger gate); made with the GIL held. A thread whose call began +// while Python was initialized and that ends it after finalization began (a daemon thread) never +// retakes the GIL: CPython before 3.13.8 would end the thread with pthread_exit, which on glibc +// unwinds into this noexcept destructor and terminates the process. It parks forever instead, as +// later CPython does. A call that began during finalization (the finalizing thread destroying +// module globals) retakes the GIL as usual. Finalization starting between the check and the +// retake surfaces as a forced unwind from PyEval_RestoreThread, which parks the thread the same way. +class Detached { + public: + Detached() : initialized_(Py_IsInitialized() != 0), state_(PyEval_SaveThread()) {} + ~Detached() { + if (initialized_ && Py_IsInitialized() == 0) park(); + attach(state_); + } + Detached(const Detached&) = delete; + Detached& operator=(const Detached&) = delete; + + private: + [[noreturn]] static void park() noexcept { + for (;;) std::this_thread::sleep_for(std::chrono::hours(1)); + } + static void attach(PyThreadState* state) { + try { + PyEval_RestoreThread(state); + } catch (...) { + park(); + } + } + bool initialized_; + PyThreadState* state_; +}; + +// Native argument work can precede computation. It uses the same detach scope, +// without announcing that the computation or its publication has begun. +template +decltype(auto) native_call(Body&& body) { + const Detached detached; + if constexpr (Computation) detail::concurrent_event(detail::ConcurrentStage::NativeBegin, nullptr); + else detail::concurrent_event(detail::ConcurrentStage::ArgumentBegin, nullptr); + if constexpr (std::is_void_v>) { + std::forward(body)(); + if constexpr (Computation) detail::publication_event(detail::ConcurrentStage::NativeComplete, nullptr); + } else { + auto result = std::forward(body)(); + if constexpr (Computation) detail::publication_event(detail::ConcurrentStage::NativeComplete, nullptr); + return result; + } +} +// Run only the graph-size part of a first pass detached, preserving the +// attached argument conversions and the surrounding check order. +inline void detached_argument_work(void (*body)(void*), void* context) { + native_call([&] { body(context); }); +} +} // namespace netgraph::core::bindings diff --git a/bindings/python/numpy_conversion.hpp b/bindings/python/numpy_conversion.hpp new file mode 100644 index 0000000..8e5270b --- /dev/null +++ b/bindings/python/numpy_conversion.hpp @@ -0,0 +1,33 @@ +// Throwing NumPy conversions, with a private allocation-test window around the cast itself. +#pragma once + +#include + +#include "netgraph/core/detail/hooks_flag.hpp" + +namespace netgraph::core::bindings { + +#if NETGRAPH_CORE_PRIVATE_HOOKS +inline thread_local bool in_numpy_conversion = false; + +class NumpyConversionScope { + public: + NumpyConversionScope() noexcept : previous_(in_numpy_conversion) { in_numpy_conversion = true; } + ~NumpyConversionScope() { in_numpy_conversion = previous_; } + NumpyConversionScope(const NumpyConversionScope&) = delete; + NumpyConversionScope& operator=(const NumpyConversionScope&) = delete; + + private: + bool previous_; +}; +#endif + +template +pybind11::array_t numpy_conversion(const pybind11::object& input) { +#if NETGRAPH_CORE_PRIVATE_HOOKS + const NumpyConversionScope scope; +#endif + return pybind11::array_t(input); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/options.cpp b/bindings/python/options.cpp new file mode 100644 index 0000000..deeec5a --- /dev/null +++ b/bindings/python/options.cpp @@ -0,0 +1,81 @@ +// The enumerations: RouteAllocation, Admission, SourceAllocation, WeightBasis, RouteTieBreak and Direction. +#include +#include + +#include + +#include "netgraph/core/options.hpp" + +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_options(py::module_& m) { + py::enum_(m, "RouteAllocation", + "How a demand's traffic is split at every router of its routes (ShortestPaths; also the interior split\n" + "of supplied PredecessorDAG routes, EQUAL or WEIGHTED).\n" + "OPTIMIZED: the minimum-cost flow core over the routes (a controller restricted to them), re-solved\n" + "over the free capacity plus the policy's own allocation (it may move its own earlier placement,\n" + "never another's). WEIGHTED (WCMP, weighted-cost multipath): every node splits in proportion to\n" + "weights fixed when the routes are prepared (highest averages (D'Hondt): assign each next unit\n" + "to the largest weight / (assigned + 1), ties to the lower EdgeId; the weights come from\n" + "WeightBasis). EQUAL (ECMP, equal-cost\n" + "multipath): every node splits equally, by round robin.") + .value("EQUAL", RouteAllocation::Equal) + .value("WEIGHTED", RouteAllocation::Weighted) + .value("OPTIMIZED", RouteAllocation::Optimized); + + py::enum_(m, "Admission", + "LOSSLESS admits the largest total the routes carry with no loss at all; LOSSY sends the whole\n" + "finite offer hop by hop and records where it is lost (every model except the optimizing ones:\n" + "ShortestPaths OPTIMIZED and CentralizedTE without tunnel_count).") + .value("LOSSLESS", Admission::Lossless) + .value("LOSSY", Admission::Lossy); + + py::enum_(m, "SourceAllocation", + "How a demand's traffic enters at its sources, separately from how it is routed. EQUAL: round\n" + "robin over the roster (ascending NodeId, the first sources one unit more); a source whose share\n" + "cannot go anywhere keeps it and loses it (LOSSLESS then admits only totals that give it\n" + "nothing). OPTIMIZED: the controller chooses the injections (one virtual source). WEIGHTED:\n" + "CentralizedTE without tunnels prescribes highest-averages shares of declared source_weights.") + .value("EQUAL", SourceAllocation::Equal) + .value("OPTIMIZED", SourceAllocation::Optimized) + .value("WEIGHTED", SourceAllocation::Weighted); + + py::enum_(m, "WeightBasis", + "Where WEIGHTED member weights come from (ShortestPaths WEIGHTED and supplied PredecessorDAG routes).\n" + "CAPACITY (the default): each member's static capacity. EXPLICIT: edge_weights, one integer per\n" + "edge (a positive weight on an edge of capacity 0 becomes 0). RESIDUAL_SNAPSHOT: each member's free\n" + "capacity read once when the routes are prepared, against one common view, and frozen for the\n" + "epoch. A member of weight 0 is pruned with the branches it leaves without a destination, inside\n" + "the route only. The tie rule never chooses weights.") + .value("CAPACITY", WeightBasis::Capacity) + .value("EXPLICIT", WeightBasis::Explicit) + .value("RESIDUAL_SNAPSHOT", WeightBasis::ResidualSnapshot); + + py::enum_(m, "RouteTieBreak", + "How a route choice breaks a cost tie; cost always comes first. FEWER_HOPS: then fewer hops,\n" + "then the smaller EdgeId sequence. HIGHER_CAPACITY / HIGHER_AVAILABLE_CAPACITY /\n" + "HIGHER_HEADROOM: then the larger path bottleneck of static capacity / availability /\n" + "availability over static capacity (exact; 0 at static capacity 0), then fewer hops, then the\n" + "EdgeId sequence. Availability: unreserved bandwidth for DistributedTE, the service view for\n" + "ShortestPaths, the residual for spf and ksp, static capacity without one. The same order\n" + "keeps one of each (u, v) pair's parallel edges when parallel_edges is False. It never\n" + "supplies weights and never enables a controller objective.") + .value("FEWER_HOPS", RouteTieBreak::FewerHops) + .value("HIGHER_CAPACITY", RouteTieBreak::HigherCapacity) + .value("HIGHER_AVAILABLE_CAPACITY", RouteTieBreak::HigherAvailableCapacity) + .value("HIGHER_HEADROOM", RouteTieBreak::HigherHeadroom); + + py::enum_(m, "Direction", + "The orientation of a search: FORWARD from the roots along the edges, or REVERSE toward the roots\n" + "against the edges (the DAG it returns keeps the edges' own direction, so its walks lead to the\n" + "roots).") + .value("FORWARD", Direction::Forward) + .value("REVERSE", Direction::Reverse); + +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/path.cpp b/bindings/python/path.cpp new file mode 100644 index 0000000..80761de --- /dev/null +++ b/bindings/python/path.cpp @@ -0,0 +1,86 @@ +// Path: an explicit walk of hops of one graph. +#include +#include +#include + +#include +#include +#include +#include + +#include "netgraph/core/graph.hpp" +#include "netgraph/core/route_structures.hpp" +#include "integers.hpp" +#include "registration.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "netgraph/core/detail/route_builders.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_path(py::module_& m) { + py::class_>( + m, "Path", + "Immutable route of one graph, hop by hop: each hop lists one or more parallel edges (members)\n" + "from the same u to the same v, in the caller's order. cost = the sum over hops of the cost of\n" + "each hop's cheapest member.") + .def(py::init([](const std::shared_ptr& graph, const py::object& hops) { + if (!graph) throw py::type_error("graph must be a Graph"); + if (is_invalid_sequence(hops)) throw py::type_error("Path: hops must be a sequence of hops (sequences of EdgeIds)"); + auto seq = py::reinterpret_borrow(hops); + std::vector raw; + for (std::size_t h = 0; h < seq.size(); ++h) { + py::object hop = seq[h]; + raw.push_back(wide_integers(hop, ("hops[" + std::to_string(h) + "]").c_str())); + } + // Every member in range before anything else (no id is narrowed unchecked), then the + // C++ constructor's order. + const auto edge_count = static_cast(graph->num_edges()); + for (std::size_t h = 0; h < raw.size(); ++h) { + for (std::size_t i = 0; i < raw[h].size(); ++i) { + if (!raw[h].in(i, 0, edge_count - 1)) { + throw py::value_error("Path: hop " + std::to_string(h) + " member " + raw[h].text(i) + + " out of edge range [0, " + std::to_string(edge_count) + ")"); + } + } + } + std::vector> narrowed; + narrowed.reserve(raw.size()); + for (const auto& r : raw) narrowed.push_back(r.narrow()); + detail::check_hops(*graph, narrowed); + return native_call([&] { return std::make_shared(detail::PathBuilder::of_hops(*graph, std::move(narrowed))); }); + }), + py::arg("graph"), py::arg("hops"), + "A wrong type raises TypeError; structural and range defects raise ValueError. Validates, in order:\n" + "every member id in range, no duplicate\n" + "member, a non-empty hop list, every hop non-empty, the members of a hop sharing (u, v),\n" + "contiguous hops, no repeated node.") + .def("graph_uid", &Path::graph_uid, "uid of the graph this path belongs to") + .def("hops", [](const Path& p) { return p.hops(); }, "the hops as lists of EdgeIds (the caller's member order)") + .def("source", &Path::source, "the first node of the walk") + .def("destination", &Path::destination, "the last node of the walk") + .def("cost", &Path::cost, "sum over hops of the minimum member cost") + .def( + "to_dag", [](const Path& p) { return native_call([&] { return std::make_shared(p.to_dag()); }); }, + "the equivalent PredecessorDAG (every member of every hop)") + .def("__eq__", [](const Path& a, const py::object& b) { + return py::isinstance(b) && a == b.cast(); + }) + .def("__hash__", + [](const Path& a) { + const auto& path_hops = a.hops(); + auto hops = py::reinterpret_steal(checked(PyTuple_New(static_cast(path_hops.size()))).release()); + for (std::size_t i = 0; i < path_hops.size(); ++i) { + PyTuple_SET_ITEM(hops.ptr(), static_cast(i), ints_tuple(path_hops[i]).release().ptr()); + } + return hash_int(objects_tuple({int_object(a.graph_uid()), hops})); + }) + .def("__repr__", [](const Path& a) { + return "Path(source=" + std::to_string(a.source()) + ", destination=" + std::to_string(a.destination()) + + ", hops=" + std::to_string(a.hops().size()) + ", cost=" + std::to_string(a.cost()) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/path_enumeration.cpp b/bindings/python/path_enumeration.cpp new file mode 100644 index 0000000..1d9b96e --- /dev/null +++ b/bindings/python/path_enumeration.cpp @@ -0,0 +1,182 @@ +// The path listings: ksp with KspOptions, and resolve_to_paths. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/enum_domain.hpp" +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/search_detail.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/options.hpp" +#include "netgraph/core/route_structures.hpp" +#include "netgraph/core/search.hpp" +#include "integers.hpp" +#include "strict_bool.hpp" +#include "publication.hpp" +#include "values.hpp" +#include "registration.hpp" +#include "native_call.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// One NodeId of [0, num_nodes) (a single endpoint); the range message of detail/endpoints.hpp. +NodeId endpoint(const py::object& obj, std::int32_t num_nodes, const char* name) { + const py::object idx = python_int(obj, name); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= num_nodes) { + throw py::value_error(detail::node_range_error(name, std::string(py::str(idx)), num_nodes)); + } + return static_cast(v); +} + +// A count of at least 1 (KSP's k, the resolver's max_paths): the validation message below 1; a count +// above INT64_MAX means the same as INT64_MAX (no list reaches it). +std::int64_t count_of(const py::object& obj, const char* name) { + const py::object idx = python_int(obj, name); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow < 0 || (overflow == 0 && v < 1)) { + throw py::value_error(std::string(name) + " must be >= 1 (got " + std::string(py::str(idx)) + ")"); + } + return overflow != 0 ? std::numeric_limits::max() : static_cast(v); +} + +// The Python form of KspOptions: the arguments as given (k, cost_ceiling and residual are +// validated at the call, in ksp's validation order, on their unnarrowed values). +struct PyKspOptions { + py::object k; + RouteTieBreak tie_break; + bool parallel_edges; + py::object cost_ceiling; + py::object residual; + std::shared_ptr eligibility; +}; + +} // namespace + +void register_paths(py::module_& m) { + py::class_( + m, "KspOptions", + "The options of ksp: k (>= 1); tie_break (RouteTieBreak) orders paths of equal cost;\n" + "parallel_edges = True lists each parallel edge as a separate path, False keeps one per (u, v)\n" + "pair by the tie rule (no multipath field: k sets the number of paths); cost_ceiling (None,\n" + "AbsoluteCost(c) with c >= 0, or CostStretch(n/d) with n/d >= 1 times the static shortest\n" + "cost under the eligibility, compared exactly, residual ignored; inclusive); residual (int64 units per\n" + "EdgeId, each in [0, edge_capacity_units]: edges with less than 1 unit are skipped, and the values\n" + "feed the availability; it is the only capacity threshold, and without it an edge of capacity 0 is\n" + "searched like any other); eligibility (None: everything allowed). A parallel_edges that is not a\n" + "bool raises TypeError here; the other fields are validated by ksp, in the order its docstring\n" + "gives.") + .def(py::init([](const py::object& k, RouteTieBreak tie_break, StrictBool parallel_edges, + const py::object& cost_ceiling, const py::object& residual, + const std::shared_ptr& eligibility) { + return PyKspOptions {k, tie_break, parallel_edges, cost_ceiling, residual, eligibility}; + }), + py::arg("k"), py::kw_only(), py::arg_v("tie_break", RouteTieBreak::FewerHops, "RouteTieBreak.FEWER_HOPS"), + py::arg("parallel_edges") = StrictBool {true}, py::arg("cost_ceiling") = py::none(), + py::arg("residual") = py::none(), py::arg("eligibility") = nullptr) + .def_readonly("k", &PyKspOptions::k, "the number of paths as given (validated by ksp: an integer >= 1)") + .def_property_readonly("tie_break", field_object(&PyKspOptions::tie_break), "the RouteTieBreak of paths of equal cost") + .def_readonly("parallel_edges", &PyKspOptions::parallel_edges, + "True: each parallel edge a separate path; False: one per (u, v) pair by the tie rule") + .def_readonly("cost_ceiling", &PyKspOptions::cost_ceiling, + "the inclusive cost ceiling as given: None, AbsoluteCost or CostStretch (validated by ksp)") + .def_readonly("residual", &PyKspOptions::residual, + "the residual units per EdgeId as given (validated by ksp); None: no capacity filter;\n" + "the tie scores read static capacity") + .def_property_readonly("eligibility", shared_field(&PyKspOptions::eligibility), "the static Eligibility; None: everything allowed"); + + m.def( + "ksp", + [](const std::shared_ptr& graph, const py::object& src, const py::object& dst, const py::object& opts) { + if (!graph) throw py::type_error("graph must be a Graph"); + const auto o = options_of(opts, "KspOptions"); + const auto n = graph->num_nodes(); + // ksp's validation order on unnarrowed values (Core repeats every check): the tie_break, the + // source, the destination, source == destination, k, the ceiling, the residual, then a foreign + // Eligibility. + detail::check_member(o.tie_break, "tie_break"); + const NodeId s = endpoint(src, n, "source"); + const NodeId t = endpoint(dst, n, "destination"); + if (s == t) throw py::value_error(detail::overlap_error(s, "source", "destination")); + KspOptions c(count_of(o.k, "k")); + c.tie_break = o.tie_break; + c.parallel_edges = o.parallel_edges; + c.cost_ceiling = cost_ceiling_of(o.cost_ceiling); + detail::check_ceiling(c.cost_ceiling); + std::optional> res; + if (!o.residual.is_none()) res = edge_units(o.residual, *graph, "residual"); + if (res.has_value()) c.residual = std::span(*res); + c.eligibility = o.eligibility.get(); + if (c.eligibility && c.eligibility->graph_uid() != graph->uid()) { + throw py::value_error("foreign Eligibility: built for another graph"); + } + auto paths = native_call([&] { return ksp(*graph, s, t, c); }); + // Every object through a checked allocation: a failure raises MemoryError. + auto out = py::reinterpret_steal(checked(PyList_New(static_cast(paths.size()))).release()); + for (std::size_t i = 0; i < paths.size(); ++i) { + py::object cost = int_object(paths[i].first); + py::object path = object_from(std::move(paths[i].second)); + PyList_SET_ITEM(out.ptr(), static_cast(i), objects_tuple({cost, path}).release().ptr()); + } + return out; + }, + py::arg("graph"), py::arg("source"), py::arg("destination"), py::arg("options"), + "The k shortest simple source -> destination paths over the eligible edges (only those with residual >= 1 when\n" + "a residual is given; an edge of capacity 0 is searched like any other) -> [(cost, Path)], each hop\n" + "a single edge. Order: lowest cost; then, under a HIGHER_* tie_break only, the highest bottleneck\n" + "(the smallest edge score along the path); then fewest hops; then the smallest EdgeId\n" + "sequence. The result is the first k paths of that order\n" + "within the inclusive cost ceiling. A wrong type raises TypeError; structural, range and combination\n" + "defects raise ValueError. Validated in this order before any work: source and\n" + "destination in range, source == destination, k >= 1, the ceiling, the residual (length, then every\n" + "entry), a foreign Eligibility. An excluded endpoint, no static route under a CostStretch ceiling, or\n" + "no path gives []."); + + m.def( + "resolve_to_paths", + [](const std::shared_ptr& graph, const std::shared_ptr& dag, const py::object& src, + const py::object& dst, StrictBool split_parallel_edges, const py::object& max_paths) { + if (!graph) throw py::type_error("graph must be a Graph"); + if (!dag) throw py::type_error("dag must be a PredecessorDAG"); + if (dag->graph_uid() != graph->uid()) throw py::value_error("foreign PredecessorDAG: built for another graph"); + const auto n = graph->num_nodes(); + const NodeId s = endpoint(src, n, "source"); + const NodeId t = endpoint(dst, n, "destination"); + if (s == t) throw py::value_error(detail::overlap_error(s, "source", "destination")); + std::optional mp; + if (!max_paths.is_none()) mp = count_of(max_paths, "max_paths"); + auto paths = native_call([&] { return resolve_to_paths(*graph, *dag, s, t, split_parallel_edges, mp); }); + auto out = py::reinterpret_steal(checked(PyList_New(static_cast(paths.size()))).release()); + for (std::size_t i = 0; i < paths.size(); ++i) { + PyList_SET_ITEM(out.ptr(), static_cast(i), object_from(std::move(paths[i])).release().ptr()); + } + return out; + }, + py::arg("graph"), py::arg("dag"), py::arg("source"), py::arg("destination"), py::arg("split_parallel_edges") = StrictBool {false}, + py::arg("max_paths") = py::none(), + "The source -> destination walks of a DAG -> [Path], in increasing (cost, hops, edge-id sequence):\n" + "a hop costs its cheapest member and is keyed by its lowest member id. split_parallel_edges = False\n" + "keeps every parallel member of a (u, v) pair in one hop (members ascending); True gives one path\n" + "per member combination, sorted by that path's own (cost, hops, edge-id sequence). max_paths\n" + "(>= 1) truncates; None lists every walk.\n" + "ValueError: a DAG of another graph, source or destination out of range, source == destination,\n" + "max_paths < 1. An endpoint outside the DAG, or no walk, gives []."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/placement_result.cpp b/bindings/python/placement_result.cpp new file mode 100644 index 0000000..d47e8e7 --- /dev/null +++ b/bindings/python/placement_result.cpp @@ -0,0 +1,347 @@ +// Termination, SourceFlow, TunnelFlow, TunnelPlan and PlacementResult: the outcome of one policy call. +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/forwarding_program.hpp" +#include "values.hpp" +#include "publication.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// A Termination's name, without a Python object (a repr must not fail on its own allocation of an +// enum object). +const char* termination_name(Termination t) { + switch (t) { + case Termination::Complete: return "COMPLETE"; + case Termination::NoRoute: return "NO_ROUTE"; + case Termination::Exhausted: return "EXHAUSTED"; + case Termination::NoActiveWeight: return "NO_ACTIVE_WEIGHT"; + } + return "?"; +} + +py::object inf_or(const std::optional& r) { + return r.has_value() ? py::object(int_object(*r)) : py::object(float_object(std::numeric_limits::infinity())); +} + +py::object counter_or_none(const std::optional& c) { + return c.has_value() ? counter_int(*c) : py::object(py::none()); +} + +// fractions.Fraction, imported once when the module is registered: TunnelPlan.max_fractional_bandwidth_per_tunnel only +// calls it (an import inside a getter can absorb a failed allocation and still succeed). +py::gil_safe_call_once_and_store& fraction_class() { + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; + return storage; +} + +} // namespace + +void register_placement_result(py::module_& m) { + fraction_class().call_once_and_store_result([] { return py::module_::import("fractions").attr("Fraction"); }); + py::enum_(m, "Termination", + "How a call ended, the first that applies: COMPLETE for a zero request; NO_ROUTE for a positive\n" + "request with an empty roster (no eligible source statically reaches an allowed destination);\n" + "NO_ACTIVE_WEIGHT when the program has an entry with a path (an up tunnel, a controller path or\n" + "a split route) but no head can use one (no positive weight: WEIGHTED over edges of capacity 0\n" + "only, routes of weight 0, every up tunnel failed); a program whose tunnels are all down is not\n" + "this case. A live CentralizedTE WEIGHTED call without tunnel_count with a positive offer\n" + "and a nonempty all-zero source-weight roster is also NO_ACTIVE_WEIGHT, with no tunnels;\n" + "Pinned replay of that empty program follows the remaining rules. A LOSSY finite request: COMPLETE (delivery and losses account for the whole offer,\n" + "which does not mean nothing was lost); otherwise COMPLETE when the whole request was delivered,\n" + "else EXHAUSTED (an infinite request is EXHAUSTED). A controller whose maximum is 0 over existing\n" + "routes is EXHAUSTED, not NO_ROUTE.") + .value("COMPLETE", Termination::Complete) + .value("NO_ROUTE", Termination::NoRoute) + .value("EXHAUSTED", Termination::Exhausted) + .value("NO_ACTIVE_WEIGHT", Termination::NoActiveWeight); + + py::class_(m, "SourceFlow", + "One normalized input source's forwarding in the epoch (read-only; zero rows included: a\n" + "masked or unreachable source): source, offered (LOSSY: its injections; None when\n" + "LOSSLESS), admitted (LOSSLESS: what it delivers; LOSSY: offered), delivered, source_drops\n" + "(offered units lost at the source for lack of a usable route) and total_edge_drops (its losses\n" + "on links). LOSSY: offered = delivered + source_drops + total_edge_drops. Exact ints.") + .def_property_readonly("source", int_reader(&SourceFlow::source), "the source node id") + .def_property_readonly( + + "offered", [](const SourceFlow& r) { return counter_or_none(r.offered); }, + "LOSSY: the units this source injected; None when LOSSLESS") + .def_property_readonly( + + "admitted", [](const SourceFlow& r) { return counter_int(r.admitted); }, + "LOSSLESS: the units it delivers; LOSSY: offered") + .def_property_readonly("delivered", int_reader(&SourceFlow::delivered), "the units it delivers to the destinations") + .def_property_readonly( + "source_drops", [](const SourceFlow& r) { return counter_int(r.source_drops); }, + "LOSSY: its offered units lost at the source (no usable route); 0 when LOSSLESS") + .def_property_readonly( + "total_edge_drops", [](const SourceFlow& r) { return counter_int(r.total_edge_drops); }, + "LOSSY: its units dropped on links; 0 when LOSSLESS") + .def("__repr__", [](const SourceFlow& r) { + return "SourceFlow(source=" + std::to_string(r.source) + ", delivered=" + std::to_string(r.delivered) + ")"; + }); + + py::class_(m, "TunnelFlow", + "One program tunnel's forwarding in the epoch (read-only): entry_id (its entry identifier) and\n" + "head, status (TunnelStatus: UP; INSUFFICIENT_BANDWIDTH or COST_CEILING_EXCEEDED for a tunnel\n" + "T1's setup left down; FAILED, BLACKHOLED in this forwarding), assigned (its share of the\n" + "injections: LOSSY the shares sent to it, LOSSLESS what it carries), delivered and\n" + "total_edge_drops (its losses on links). Exact ints (assigned and total_edge_drops may exceed 64\n" + "bits).") + .def_property_readonly("entry_id", int_reader(&TunnelFlow::entry_id), "the tunnel's entry identifier") + .def_property_readonly("head", int_reader(&TunnelFlow::head), "the tunnel's head") + .def_property_readonly("status", field_object(&TunnelFlow::status), + "UP, a down tunnel's setup status (INSUFFICIENT_BANDWIDTH, COST_CEILING_EXCEEDED), FAILED or\n" + "BLACKHOLED in this forwarding") + .def_property_readonly( + + "assigned", [](const TunnelFlow& r) { return counter_int(r.assigned); }, + "its share of the injections in the epoch (LOSSY: sent to it; LOSSLESS: what it carries)") + .def_property_readonly("delivered", int_reader(&TunnelFlow::delivered), "the units it delivers") + .def_property_readonly( + "total_edge_drops", [](const TunnelFlow& r) { return counter_int(r.total_edge_drops); }, "its units dropped on links") + .def("__repr__", [](const TunnelFlow& r) { + return "TunnelFlow(entry_id=" + std::to_string(r.entry_id) + ", delivered=" + std::to_string(r.delivered) + ")"; + }); + + py::class_(m, "TunnelPlan", + "The tunnel models' setup summary (read-only): tunnels_per_head (T2 up to: the chosen count),\n" + "design_demand (the design demand D in units), design_by_head ((head, RR(D, H, i)) for every\n" + "roster head in ascending NodeId) and total_reserved_bandwidth (T1's strict holds, counted once\n" + "per up tunnel: the sum of their requests; 0 for T2). T2 only (None for T1): heads_with_capacity,\n" + "insufficient_resolution (live heads exist but no whole unit per tunnel fits, U == 0: the\n" + "program has no tunnels), max_fractional_bandwidth_per_tunnel (the exact fluid optimum s*, a\n" + "Fraction), max_bandwidth_per_tunnel (U), planned_bandwidth_per_tunnel (u),\n" + "total_planned_bandwidth (K*u), total_tunnel_cost (the route-cost sum, can exceed 2^64),\n" + "headroom_penalty (with prefer_headroom only) and max_lossless_offer (the program's lossless\n" + "maximum on the view it was planned on; a dead head's share blocks it). Exact ints. A plan\n" + "describes the computation that installed the tunnels: forward returns it unchanged, its\n" + "bounds not recomputed for the new offer, load or black holes.") + .def_property_readonly("tunnels_per_head", int_reader(&TunnelPlan::tunnels_per_head), "the tunnels per head (T2 up to: the chosen count)") + .def_property_readonly( + + "design_demand", [](const TunnelPlan& p) { return inf_or(p.design_demand); }, + "the design volume D in units; float('inf') when unlimited") + .def_property_readonly( + "design_by_head", [](const TunnelPlan& p) { return pairs_tuple(p.design_by_head); }, + "((head, share), ...): every roster head's share RR(D, H, i), ascending NodeId") + .def_property_readonly("total_reserved_bandwidth", int_reader(&TunnelPlan::total_reserved_bandwidth), + "the strict holds of the up tunnels, each tunnel's request counted once") + .def_property_readonly( + "heads_with_capacity", + [](const TunnelPlan& p) -> py::object { + return p.heads_with_capacity.has_value() ? py::object(ints_tuple(*p.heads_with_capacity)) : py::object(py::none()); + }, + "T2: the heads that reach a destination over links with free capacity >= 1 (ascending); None for T1") + .def_readonly("insufficient_resolution", &TunnelPlan::insufficient_resolution, + "T2: True when live heads exist but no whole unit per tunnel fits (max_bandwidth_per_tunnel 0,\n" + "no tunnels)") + .def_property_readonly( + "max_fractional_bandwidth_per_tunnel", + [](const TunnelPlan& p) -> py::object { + if (!p.max_fractional_bandwidth_per_tunnel.has_value()) return py::none(); + const auto& [n, d] = *p.max_fractional_bandwidth_per_tunnel; + return fraction_object(fraction_class().get_stored(), n, d); + }, + "T2: the exact fluid optimum s* (a fractions.Fraction; 0 without a live head); None for T1") + .def_property_readonly("max_bandwidth_per_tunnel", optional_int_reader(&TunnelPlan::max_bandwidth_per_tunnel), "T2: U, the largest common whole-unit size") + .def_property_readonly("planned_bandwidth_per_tunnel", optional_int_reader(&TunnelPlan::planned_bandwidth_per_tunnel), "T2: u, every tunnel's planned bandwidth") + .def_property_readonly("total_planned_bandwidth", optional_int_reader(&TunnelPlan::total_planned_bandwidth), "T2: K*u over the K tunnels") + .def_property_readonly( + "total_tunnel_cost", + [](const TunnelPlan& p) -> py::object { return p.total_tunnel_cost.has_value() ? counter_int(*p.total_tunnel_cost) : py::none(); }, + "T2: the sum of cost * tunnel count over the links (exact; can exceed 2^64)") + .def_property_readonly( + + "headroom_penalty", + [](const TunnelPlan& p) -> py::object { + return p.headroom_penalty.has_value() ? counter_int(*p.headroom_penalty) : py::none(); + }, + "T2 with prefer_headroom: the sum over the links' tunnel slots j of floor(2^32 (B - V + j*u) / B)\n" + "(B: static capacity; V: free capacity); None without prefer_headroom and for T1") + .def_property_readonly("max_lossless_offer", optional_int_reader(&TunnelPlan::max_lossless_offer), + "T2: F, the program's lossless maximum on the view it was planned on (a dead head's share blocks\n" + "it); forward keeps the plan unchanged") + .def("__repr__", [](const TunnelPlan& p) { + return "TunnelPlan(tunnels_per_head=" + std::to_string(p.tunnels_per_head) + ", total_reserved_bandwidth=" + std::to_string(p.total_reserved_bandwidth) + ")"; + }); + + py::class_( + m, "PlacementResult", + "The result of one call. This call: call_delivered (units delivered by the call; replace, forward\n" + "and evaluate: the full new delivery), call_undelivered (request - call_delivered; float('inf') for\n" + "an infinite request), delivery_delta (new minus old delivery, may be negative), call_edge_drops\n" + "[(EdgeId, units)], call_source_drops, termination, routes_changed, weights_changed and\n" + "reservations_changed. The\n" + "demand's state after the call (totals of the epoch): program (the installed ForwardingProgram;\n" + "evaluate: the computed one; None before the first preparation), plan (a TunnelPlan for T1 and T2;\n" + "None for every other model), requested (float('inf') after an infinite request), offered (LOSSY: the traffic sent; None when LOSSLESS), admitted, delivered, unadmitted\n" + "(LOSSLESS: requested - delivered, None after an infinite request; LOSSY: 0), source_drops,\n" + "edge_drops [(EdgeId, units)], by_source (one SourceFlow per normalized input source, ascending\n" + "NodeId), by_tunnel (one TunnelFlow per program tunnel, ascending entry_id; an alternative view, never\n" + "added to by_source), total_cost (controllers: CentralizedTE and ShortestPaths OPTIMIZED; else None)\n" + "and marginal_cost_curve (unrestricted CentralizedTE with the optimized source allocation only).\n" + "LOSSY: offered = source_drops + sum(edge_drops) + delivered. The rows sum to the totals, except, under\n" + "LOSSY, for an empty roster and for a forwarding whose remaining captured source weights are all 0\n" + "(a controller program that sent nothing, or a mask that left only zero-weight heads): the whole\n" + "offer is then the demand's source drops and every row is zero. Optional outputs, None unless requested: edge_flows, residual and edge_reservations\n" + "(int64[E]), saturated_boundary_edges, min_cut (they describe the same\n" + "state; evaluate: the evaluated one). Amounts that accumulate are exact ints (they may exceed 64 bits).") + .def_property_readonly("call_delivered", int_reader(&PlacementResult::call_delivered), "the units placed (delivered) by this call") + .def_property_readonly( + "call_undelivered", [](const PlacementResult& r) { return inf_or(r.call_undelivered); }, + "the request minus call_delivered, in units (an amount, not a status); float('inf') for an infinite\n" + "request") + .def_property_readonly("delivery_delta", int_reader(&PlacementResult::delivery_delta), + "the delivery after the call minus before it (call_delivered for a placement; may be negative\n" + "for a full-offer call)") + .def_property_readonly( + "call_edge_drops", [](const PlacementResult& r) { return pairs_list(r.call_edge_drops); }, + "this call's drops on links [(EdgeId, units)], ascending EdgeId (empty unless LOSSY)") + .def_property_readonly("call_source_drops", int_reader(&PlacementResult::call_source_drops), + "this call's offered units lost at sources without a usable route (LOSSY)") + .def_property_readonly("termination", field_object(&PlacementResult::termination), "the call's outcome (Termination)") + .def_readonly("routes_changed", &PlacementResult::routes_changed, + "True when the call's program differs from the installed one in its ordered heads, paths, split\n" + "tables or duplicates (no program counts as the empty program; two programs without an entry\n" + "are equal). An offer-only change (forward, Pinned, ShortestPaths increments) and an identical\n" + "replace set nothing") + .def_readonly("weights_changed", &PlacementResult::weights_changed, + "True when the call's program differs from the installed one in its injection (source weights)\n" + "or any entry or split-table weight (a controller's re-solve that changes a path's amount; an\n" + "absent entry weighs 0)") + .def_readonly("reservations_changed", &PlacementResult::reservations_changed, + "True when a tunnel's requested or reserved amount differs from the installed program's, by\n" + "entry_id (an absent tunnel holds 0): only T1 sets it; forward and Pinned never do") + .def_property_readonly( + + "program", + [](const PlacementResult& r) { return optional_shared(r.program); }, + "the installed ForwardingProgram after the call (evaluate: the computed one); None before the first\n" + "preparation") + .def_property_readonly( + + "plan", [](const PlacementResult& r) { return optional_of(r.plan); }, + "T1 and T2: the TunnelPlan of the program after the call; None for every other model (Pinned over\n" + "a tunnel program too)") + .def_property_readonly( + + "requested", + [](const PlacementResult& r) -> py::object { + return r.requested.has_value() ? counter_int(*r.requested) + : py::object(float_object(std::numeric_limits::infinity())); + }, + "the sum of the epoch's requests; float('inf') after an infinite one") + .def_property_readonly( + + "offered", [](const PlacementResult& r) { return counter_or_none(r.offered); }, + "LOSSY: the traffic sent in the epoch (the first-hop injection of an infinite request); None\n" + "when LOSSLESS") + .def_property_readonly( + + "admitted", [](const PlacementResult& r) { return counter_int(r.admitted); }, + "LOSSLESS: the total carried; LOSSY: offered") + .def_property_readonly("delivered", int_reader(&PlacementResult::delivered), "the units the demand delivers") + .def_property_readonly( + + "unadmitted", [](const PlacementResult& r) { return counter_or_none(r.unadmitted); }, + "LOSSLESS: requested - delivered, None after an infinite request; LOSSY: 0") + .def_property_readonly( + "source_drops", [](const PlacementResult& r) { return counter_int(r.source_drops); }, + "LOSSY: the offered units lost at sources (no usable route, or an empty roster); 0 when LOSSLESS") + .def_property_readonly( + "edge_drops", [](const PlacementResult& r) { return pairs_list(r.edge_drops); }, + "the epoch's drops on links [(EdgeId, units)], ascending EdgeId (empty unless LOSSY)") + .def_property_readonly( + + "by_source", [](const PlacementResult& r) { return tuple_of(r.by_source); }, + "one SourceFlow per normalized input source, ascending NodeId (zero rows included)") + .def_property_readonly( + + "by_tunnel", [](const PlacementResult& r) { return tuple_of(r.by_tunnel); }, + "one TunnelFlow per program tunnel, ascending entry_id (empty without tunnels)") + .def_property_readonly( + + "total_cost", [](const PlacementResult& r) { return counter_or_none(r.total_cost); }, + "controllers (CentralizedTE, ShortestPaths OPTIMIZED): the exact sum of cost * flow over the policy's\n" + "allocation; None for the other models (T1, T2, Pinned)") + .def_property_readonly( + "marginal_cost_curve", + [](const PlacementResult& r) { + using Curve = py::typing::List>; + return py::reinterpret_steal>( + r.marginal_cost_curve ? pairs_list(*r.marginal_cost_curve).release() : py::none().release()); + }, + "unrestricted CentralizedTE with the optimized source allocation: the marginal cost curve [(cost, units)] of the policy's\n" + "flow as the call's solve computed it (a positive place: the cumulative flow; evaluate and replace,\n" + "a zero offer included: the whole offer); place(0) keeps the stored curve (None before the epoch's\n" + "first solve or forward; after a forward, None for a nonempty roster and [] for an empty one);\n" + "a forward with a nonempty roster solves nothing and reports None; with an empty roster every\n" + "call but place(0) reports []; in ascending cost, delivering `units` more raised the minimum total flow cost by `cost`\n" + "per unit (more delivery can reroute earlier flow, so these are not the costs of the final\n" + "paths). None for the other models and prescribed (EQUAL or WEIGHTED) sources") + .def_property_readonly( + + "edge_flows", + [](const PlacementResult& r) -> py::object { + return r.edge_flows ? py::object(int64_array(*r.edge_flows)) : py::object(py::none()); + }, + "if requested: the policy's own allocation per edge after the call (int64[E], EdgeId order),\n" + "else None") + .def_property_readonly( + + "residual", + [](const PlacementResult& r) -> py::object { + return r.residual ? py::object(int64_array(*r.residual)) : py::object(py::none()); + }, + "if requested: the ledger's residual after the call (int64[E], EdgeId order), else None") + .def_property_readonly( + "edge_reservations", + [](const PlacementResult& r) -> py::object { + return r.edge_reservations ? py::object(int64_array(*r.edge_reservations)) : py::object(py::none()); + }, + "if requested: the policy's own strict holds per edge after the call (int64[E], EdgeId order; zeros\n" + "for a model without holds), else None") + .def_property_readonly( + "saturated_boundary_edges", + [](const PlacementResult& r) { + return py::reinterpret_steal>>( + r.saturated_boundary_edges ? ints_list(std::span(*r.saturated_boundary_edges)).release() : py::none().release()); + }, + "if requested: the ledger's saturation report after the call\n" + "(FlowLedger.saturated_boundary_edges of the " + "policy's sources and Eligibility), else None") + .def_property_readonly( + "min_cut", + [](const PlacementResult& r) { + return py::reinterpret_steal>>( + r.min_cut ? ints_list(std::span(*r.min_cut)).release() : py::none().release()); + }, + "if requested (OutputOptions(min_cut=True)), unrestricted CentralizedTE with the optimized source allocation, when the solve\n" + "reached the maximum flow: the minimum cut closest to the sources, every allowed edge leaving\n" + "the nodes the roster sources reach in the residual network (free capacity plus the policy's\n" + "own allocation), an edge with nothing free (capacity 0, or held by others) included,\n" + "ascending EdgeId; [] for an empty roster, place(0) excepted. None otherwise (another model,\n" + "prescribed EQUAL or WEIGHTED sources, a support, a solve stopped below the maximum, place(0),\n" + "a forward with a nonempty roster)") + .def("__repr__", [](const PlacementResult& r) { + return "PlacementResult(call_delivered=" + std::to_string(r.call_delivered) + ", termination=" + + termination_name(r.termination) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/policy_value.hpp b/bindings/python/policy_value.hpp new file mode 100644 index 0000000..6fe66e8 --- /dev/null +++ b/bindings/python/policy_value.hpp @@ -0,0 +1,23 @@ +// Reject an unconstructed Python policy before pybind11's ordinary value loader +// can allocate raw storage for it. Explicit __new__/failed __init__ is legal Python. +#pragma once +#include +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/detail/policy_engine.hpp" + +namespace netgraph::core::bindings { +inline FlowPolicy& policy_value(const pybind11::handle& self) { + const auto* type = pybind11::detail::get_type_info(typeid(FlowPolicy), true); + // isinstance can consult a forged __class__. Only the real extension type has + // the instance layout read below; never reinterpret an unrelated Python object. + if (PyObject_TypeCheck(self.ptr(), type->type) == 0) throw pybind11::type_error("policy must be a FlowPolicy"); + auto* instance = reinterpret_cast(self.ptr()); + auto value = instance->get_value_and_holder(type); + if (!value.holder_constructed() || !value.instance_registered() || value.value_ptr() == nullptr) { + throw pybind11::type_error("FlowPolicy is not constructed"); + } + auto& policy = *static_cast(value.value_ptr()); + if (!detail::PolicyAccess::constructed(policy)) throw pybind11::type_error("FlowPolicy is not constructed"); + return policy; +} +} // namespace netgraph::core::bindings diff --git a/bindings/python/predecessor_dag.cpp b/bindings/python/predecessor_dag.cpp new file mode 100644 index 0000000..e2a1b57 --- /dev/null +++ b/bindings/python/predecessor_dag.cpp @@ -0,0 +1,130 @@ +// PredecessorDAG: the immutable predecessor DAG of one graph. +#include +#include +#include + +#include +#include +#include + +#include "netgraph/core/graph.hpp" +#include "netgraph/core/route_structures.hpp" +#include "integers.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "netgraph/core/detail/route_builders.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The exact order of wide integers: a positive overflow above every int64, a negative one +// below, and two overflows on the same side by their exact values. +bool wide_less(const CheckedIntegerVector& w, std::size_t i, std::size_t j) { + if (w.sign(i) != w.sign(j)) return w.sign(i) < w.sign(j); + if (w.sign(i) == 0) return w.value[i] < w.value[j]; + const int less = PyObject_RichCompareBool(w.exact(i).ptr(), w.exact(j).ptr(), Py_LT); + if (less < 0) throw py::error_already_set(); + return less == 1; +} + +} // namespace + +void register_predecessor_dag(py::module_& m) { + py::class_>( + m, "PredecessorDAG", + "Immutable route structure of one graph, a predecessor DAG (directed, no cycle): for each node v,\n" + "the edges that reach v on the route and the node each comes from (its parent). The entries of v\n" + "are parents()[k], via_edges()[k] for k in [parent_offsets()[v], parent_offsets()[v + 1]), in\n" + "strictly ascending EdgeId, so two DAGs of the same graph with the same entries have equal arrays\n" + "and compare equal; DAGs of different graphs never compare equal.\n" + "parents() and via_edges() are read-only views that keep the DAG alive; parent_offsets() is a new\n" + "read-only array on each call (the DAG stores only the nodes that have entries). Built by\n" + "PredecessorDAG.from_csr, Path.to_dag(), the searches and the policies (FlowRecord.route).") + .def_static( + "from_csr", + [](const std::shared_ptr& graph, const py::object& offsets, const py::object& parents, + const py::object& via) { + if (!graph) throw py::type_error("graph must be a Graph"); + // Validate the unnarrowed shape and ranges, then the native structural + // rules below (endpoints, duplicates, acyclic, ascending), all before + // detaching. The builder consumes these checked native arrays. + const auto n = static_cast(graph->num_nodes()); + const auto edge_count = static_cast(graph->num_edges()); + const std::size_t lo = sequence_length(offsets, "parent_offsets"); + const std::size_t lp = sequence_length(parents, "parents"); + const std::size_t lv = sequence_length(via, "via_edges"); + if (lo != n + 1) { + throw py::value_error("PredecessorDAG.from_csr: parent_offsets has length " + std::to_string(lo) + + ", expected N + 1 = " + std::to_string(n + 1)); + } + const auto o = wide_integers(offsets, "parent_offsets"); + if (!o.in(0, 0, 0)) throw py::value_error("PredecessorDAG.from_csr: parent_offsets[0] = " + o.text(0) + ", expected 0"); + for (std::size_t v = 0; v < n; ++v) { + if (wide_less(o, v + 1, v)) { + throw py::value_error("PredecessorDAG.from_csr: parent_offsets are not monotone at " + std::to_string(v + 1) + " (" + + o.text(v) + " > " + o.text(v + 1) + ")"); + } + } + if (!o.in(n, static_cast(lp), static_cast(lp)) || lp != lv) { + throw py::value_error("PredecessorDAG.from_csr: parent_offsets[N] = " + o.text(n) + " but parents has " + + std::to_string(lp) + " and via_edges has " + std::to_string(lv) + " entries"); + } + const auto p = wide_integers(parents, "parents"); + const auto w = wide_integers(via, "via_edges"); + for (std::size_t k = 0; k < lp; ++k) { + if (!p.in(k, 0, static_cast(n) - 1)) { + throw py::value_error("PredecessorDAG.from_csr: parents[" + std::to_string(k) + "] = " + p.text(k) + + " out of node range [0, " + std::to_string(n) + ")"); + } + if (!w.in(k, 0, edge_count - 1)) { + throw py::value_error("PredecessorDAG.from_csr: via_edges[" + std::to_string(k) + "] = " + w.text(k) + + " out of edge range [0, " + std::to_string(edge_count) + ")"); + } + } + const auto ov = o.narrow(); + const auto pv = p.narrow(); + const auto wv = w.narrow(); + detail::check_csr(*graph, ov, pv, wv); + return native_call([&] { return std::make_shared(detail::PredecessorDAGBuilder::of_csr(*graph, ov, pv, wv)); }); + }, + py::arg("graph"), py::arg("parent_offsets"), py::arg("parents"), py::arg("via_edges"), + "A wrong type raises TypeError; structural and range defects raise ValueError. Validates, in order,\n" + "before any traversal: parent_offsets of\n" + "length N + 1, parent_offsets[0] == 0, monotone, parent_offsets[N] == len(parents) ==\n" + "len(via_edges); then every parent and via edge in range, every via edge joining (parent, child),\n" + "no duplicate (child, via edge), acyclic; within a node the entries in ascending via EdgeId.") + .def("graph_uid", &PredecessorDAG::graph_uid, "uid of the graph this DAG belongs to") + .def( + "parent_offsets", + [](const PredecessorDAG& d) { + const auto off = d.parent_offsets(); + py::array_t arr(static_cast(off.size())); + std::copy(off.begin(), off.end(), arr.mutable_data()); + py::detail::array_proxy(arr.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + return arr; + }, + "int32[N + 1]: the entries of node v are [parent_offsets[v], parent_offsets[v + 1]); a new read-only array per call") + .def( + "parents", [](const py::object& self) { return readonly_view(self.cast().parents(), self); }, + "int32 parent NodeId per entry (read-only view)") + .def( + "via_edges", + [](const py::object& self) { return readonly_view(self.cast().via_edges(), self); }, + "int32 via EdgeId per entry (read-only view)") + .def("__eq__", [](const PredecessorDAG& a, const py::object& b) { + return py::isinstance(b) && a == b.cast(); + }) + .def("__hash__", [](const PredecessorDAG& a) { + return hash_int(objects_tuple({int_object(a.graph_uid()), ints_tuple(a.via_edges())})); + }) + .def("__repr__", [](const PredecessorDAG& a) { + return "PredecessorDAG(graph_uid=" + std::to_string(a.graph_uid()) + ", entries=" + + std::to_string(a.via_edges().size()) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/publication.hpp b/bindings/python/publication.hpp new file mode 100644 index 0000000..f00b209 --- /dev/null +++ b/bindings/python/publication.hpp @@ -0,0 +1,218 @@ +/* + The checked construction of result objects: the Python object of a bound class is allocated + through a checked tp_alloc, so an allocation failure raises instead of crashing, and a policy + call makes its result object before it changes anything. +*/ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/evaluators.hpp" +#include "netgraph/core/flow_policy.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// A new Python object of the bound class T owning a T made from `args`, made like pybind11's +// generic move-cast (type_caster_generic::cast) but with the object allocation checked: +// pybind11's make_new_instance uses the result of tp_alloc unchecked, so a failed Python +// allocation there is a crash rather than a MemoryError. Holder is the class's holder type +// (std::unique_ptr by default, std::shared_ptr for PredecessorDAG and Path); another type is +// refused before anything is made. +// The steps, in order (new_object and object_from both use them): (1) allocate the object and +// check for null; (2) hold it in `obj` at once, so any later failure releases it; (3) allocate +// its layout; (4) make the value inside a local holder; (5) register the value and let the +// object take the holder; (6) mark the object as owner. Every failure raises and frees what was +// made exactly once. Until step 5 succeeds the object owns nothing: the local holder frees the +// value (std::shared_ptr also deletes it when its control block cannot be allocated), and the +// object is released without touching it. Taking the holder never allocates (pybind11 copies or +// moves it). +template +std::pair emplace_object(Args&&... args) { + static_assert(std::is_same_v> || std::is_same_v>); + const py::detail::type_info* tinfo = py::detail::get_type_info(typeid(T), /*throw_if_missing=*/true); + constexpr auto holder_kind = std::is_same_v> ? py::detail::holder_enum_t::std_shared_ptr + : py::detail::holder_enum_t::std_unique_ptr; + if (tinfo->holder_enum_v != holder_kind) { + py::pybind11_fail("emplace_object: Holder is not the holder of the bound class"); + } + PyTypeObject* type = tinfo->type; + PyObject* raw = type->tp_alloc(type, 0); + if (raw == nullptr) throw py::error_already_set(); + auto obj = py::reinterpret_steal(raw); + auto* inst = reinterpret_cast(raw); + inst->allocate_layout(); + inst->owned = false; + Holder holder(new T(std::forward(args)...)); + T* value = holder.get(); + py::detail::values_and_holders(inst).begin()->value_ptr() = value; + tinfo->init_instance(inst, &holder); + inst->owned = true; + return {std::move(obj), value}; +} + +// A new Python object of the bound class T (default holder) owning a default-constructed T +// (emplace_object). The policy calls make their result object with this before they change +// anything. +template +std::pair new_object() { + return emplace_object>(); +} + +// The value moved into its result object after the commit: nothing may allocate there. +static_assert(std::is_nothrow_move_assignable_v); +static_assert(std::is_nothrow_move_assignable_v); +static_assert(std::is_nothrow_move_assignable_v); + +template +py::object object_of(T value) { + return emplace_object>(std::move(value)).first; +} + +// A tuple of new Python objects of the bound class T, one per row (object_of each): a failed +// allocation raises and releases what was made (pybind11's cast of a vector of bound values +// allocates each object unchecked). +template +py::tuple tuple_of(const std::vector& rows) { + PyObject* raw = PyTuple_New(static_cast(rows.size())); + if (raw == nullptr) throw py::error_already_set(); + auto out = py::reinterpret_steal(raw); + for (std::size_t i = 0; i < rows.size(); ++i) { + PyTuple_SET_ITEM(out.ptr(), static_cast(i), object_of(rows[i]).release().ptr()); + } + return out; +} + +// A new Python object of the bound class T (held by std::shared_ptr) owning `value`, through the +// same checked allocation as new_object: for the values without a default constructor (PredecessorDAG, +// Path) that a search returns. A failure raises and releases everything made so far. +template +py::object object_from(T value) { + return emplace_object>(std::move(value)).first; +} + +// The Python object of a shared value of the bound class T (held by std::shared_ptr): None for a +// null pointer; the object already registered for it, if any (as pybind11's cast); else a new +// object made by emplace_object's steps with `holder` in place of a new value. pybind11's own +// cast of an existing holder marks the object as owner before it registers the value, so a +// failed registration deletes the value while the holder still owns it (a double free); here +// the object owns nothing until it holds its copy of the holder (a copy never allocates). +template +py::object shared_object(std::shared_ptr holder) { + if (!holder) return py::none(); + const py::detail::type_info* tinfo = py::detail::get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v != py::detail::holder_enum_t::std_shared_ptr) { + py::pybind11_fail("shared_object: the bound class is not held by std::shared_ptr"); + } + if (py::handle existing = py::detail::find_registered_python_instance(holder.get(), tinfo)) { + return py::reinterpret_steal(existing); + } + PyTypeObject* type = tinfo->type; + PyObject* raw = type->tp_alloc(type, 0); + if (raw == nullptr) throw py::error_already_set(); + auto obj = py::reinterpret_steal(raw); + auto* inst = reinterpret_cast(raw); + inst->allocate_layout(); + inst->owned = false; + py::detail::values_and_holders(inst).begin()->value_ptr() = holder.get(); + tinfo->init_instance(inst, &holder); + inst->owned = true; + return obj; +} + +template +py::object shared_object(std::shared_ptr holder) { + return shared_object(std::const_pointer_cast(std::move(holder))); +} + +// A py::object known to be a T, for signatures: pybind11 names it as T (the specialization of +// handle_type_name below), so a binding that returns shared_object's result keeps its type. +template +class TypedPythonObject : public py::object { +public: + TypedPythonObject() = default; + TypedPythonObject(py::handle h, py::object::stolen_t) : py::object(h, py::object::stolen_t {}) {} + TypedPythonObject(py::handle h, py::object::borrowed_t) : py::object(h, py::object::borrowed_t {}) {} + static bool check_(py::handle h) { return py::isinstance(h); } +}; + +// object_of(value) as a T; a list of object_of each row (as tuple_of); object_of of a present +// value or None. pybind11's cast of a bound value, a vector or an optional of them allocates the +// object unchecked. +template +TypedPythonObject typed_object(T value) { + return py::reinterpret_steal>(object_of(std::move(value)).release()); +} +template +py::typing::List list_of(const std::vector& rows) { + PyObject* raw = PyList_New(static_cast(rows.size())); + if (raw == nullptr) throw py::error_already_set(); + auto out = py::reinterpret_steal>(raw); + for (std::size_t i = 0; i < rows.size(); ++i) { + PyList_SET_ITEM(out.ptr(), static_cast(i), object_of(rows[i]).release().ptr()); + } + return out; +} +template +py::typing::Optional optional_of(const std::optional& value) { + return py::reinterpret_steal>(value.has_value() ? object_of(*value).release() + : py::none().release()); +} + +// The repr of a bound value through its class's own __repr__. PyObject_Repr (py::repr) on Python +// 3.13 and older returns the default repr when the lookup of the method fails to allocate. +inline std::string own_repr(const py::handle& value) { + return std::string(py::str(py::type::of(value).attr("__repr__")(value))); +} + +// shared_object(holder) as a T (holder not null) or as T | None. +template +TypedPythonObject> typed_shared(std::shared_ptr holder) { + return py::reinterpret_steal>>(shared_object(std::move(holder)).release()); +} +template +py::typing::Optional> optional_shared(std::shared_ptr holder) { + return py::reinterpret_steal>>(shared_object(std::move(holder)).release()); +} + +// The getters of read-only fields whose values are Python objects of bound classes or +// enumerations: def_readonly casts them through pybind11's unchecked allocation (a failed Python +// allocation is a crash); these make them through the checked one above. A field of a bound value +// or enumeration, an optional one, a shared one (its registered object, or None) and a vector of +// bound values (a list). +template +auto field_object(D C::*member) { + return [member](const C& c) { return typed_object(c.*member); }; +} +template +auto optional_field(std::optional C::*member) { + return [member](const C& c) { return optional_of(c.*member); }; +} +template +auto shared_field(std::shared_ptr C::*member) { + return [member](const C& c) { return optional_shared(c.*member); }; +} +template +auto list_field(std::vector C::*member) { + return [member](const C& c) { return list_of(c.*member); }; +} + +} // namespace netgraph::core::bindings + +namespace pybind11::detail { +template +struct handle_type_name> { + static constexpr auto name = make_caster::name; +}; +} // namespace pybind11::detail diff --git a/bindings/python/registration.hpp b/bindings/python/registration.hpp new file mode 100644 index 0000000..f479c9a --- /dev/null +++ b/bindings/python/registration.hpp @@ -0,0 +1,43 @@ +/* + The registration functions of the module _netgraph_core, one file each under bindings/python/. + The module entry (module.cpp) calls them in this order, which is the order of the module's + attributes: a default argument or a conversion of a later binding needs the types registered + before it. The private recorder entry (_solve_min_cost_flow) is registered + in both builds; only register_private_hooks and the private members of FlowLedger and + FlowPolicy depend on NETGRAPH_CORE_PRIVATE_HOOKS. +*/ +#pragma once + +#include +#include +#include + +#include "netgraph/core/detail/hooks_flag.hpp" + +namespace netgraph::core::bindings { + +void register_options(pybind11::module_& m); // the axes' enumerations +void register_units(pybind11::module_& m); // to_units, as_float +void register_eligibility(pybind11::module_& m); // Eligibility +void register_graph(pybind11::module_& m); // Graph +void register_flow_solver_hook(pybind11::module_& m); // _solve_min_cost_flow (both builds) +void register_predecessor_dag(pybind11::module_& m); // PredecessorDAG +void register_path(pybind11::module_& m); // Path +void register_flow_record(pybind11::module_& m); // FlowRecord +void register_flow_ledger(pybind11::module_& m); // FlowLedger +void register_program(pybind11::module_& m); // TunnelStatus, Tunnel, RouteDefinition, ForwardingProgram +void register_routing(pybind11::module_& m); // AbsoluteCost, CostStretch, TunnelCount, the four routings +void register_policy_config(pybind11::module_& m); // FlowPolicyConfig, OutputOptions +void register_placement_result(pybind11::module_& m); // Termination, SourceFlow, TunnelFlow, PlacementResult +void register_shortest_path_cache(pybind11::module_& m); // ShortestPathCacheStats, ShortestPathCacheMiss, ShortestPathCache +void register_flow_policy(pybind11::module_& m); // FlowPolicy +void register_apply_actions(pybind11::module_& m); // BatchMode, Place, Replace, Forward, apply_actions +void register_evaluators(pybind11::module_& m); // evaluate_demand, evaluate_pairs, evaluate_edge_failures and their types +void register_paths(pybind11::module_& m); // KspOptions, ksp, resolve_to_paths +void register_search(pybind11::module_& m); // UNREACHABLE, SpfOptions, SpfResult, spf +#if NETGRAPH_CORE_PRIVATE_HOOKS +void register_concurrency_hooks(pybind11::module_& m); +void register_private_hooks(pybind11::module_& m); // the test controls and diagnostics +#endif + +} // namespace netgraph::core::bindings diff --git a/bindings/python/routing_models.cpp b/bindings/python/routing_models.cpp new file mode 100644 index 0000000..aa1eabe --- /dev/null +++ b/bindings/python/routing_models.cpp @@ -0,0 +1,432 @@ +// The routing modes of a policy (ShortestPaths, DistributedTE, CentralizedTE, Pinned) and their parameters: +// the cost ceilings AbsoluteCost and CostStretch, the tunnel count TunnelCount, CentralizedTE's support and Pinned's +// program and black holes. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/options.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/route_structures.hpp" +#include "netgraph/core/forwarding_program.hpp" +#include "integers.hpp" +#include "strict_bool.hpp" +#include "publication.hpp" +#include "registration.hpp" +#include "values.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// A RouteTieBreak as its repr spells it, without a Python object (a repr must not fail on its own +// allocations of an enum object). +const char* tie_break_repr(RouteTieBreak t) { + switch (t) { + case RouteTieBreak::FewerHops: return "RouteTieBreak.FEWER_HOPS"; + case RouteTieBreak::HigherCapacity: return "RouteTieBreak.HIGHER_CAPACITY"; + case RouteTieBreak::HigherAvailableCapacity: return "RouteTieBreak.HIGHER_AVAILABLE_CAPACITY"; + case RouteTieBreak::HigherHeadroom: return "RouteTieBreak.HIGHER_HEADROOM"; + } + return "RouteTieBreak(?)"; +} + +// The other enumerations of the models as their reprs spell them (C++ tables, as tie_break_repr). +const char* allocation_repr(RouteAllocation a) { + switch (a) { + case RouteAllocation::Equal: return "RouteAllocation.EQUAL"; + case RouteAllocation::Weighted: return "RouteAllocation.WEIGHTED"; + case RouteAllocation::Optimized: return "RouteAllocation.OPTIMIZED"; + } + return "RouteAllocation(?)"; +} +const char* source_allocation_repr(SourceAllocation a) { + switch (a) { + case SourceAllocation::Equal: return "SourceAllocation.EQUAL"; + case SourceAllocation::Optimized: return "SourceAllocation.OPTIMIZED"; + case SourceAllocation::Weighted: return "SourceAllocation.WEIGHTED"; + } + return "SourceAllocation(?)"; +} +const char* weight_basis_repr(WeightBasis b) { + switch (b) { + case WeightBasis::Capacity: return "WeightBasis.CAPACITY"; + case WeightBasis::Explicit: return "WeightBasis.EXPLICIT"; + case WeightBasis::ResidualSnapshot: return "WeightBasis.RESIDUAL_SNAPSHOT"; + } + return "WeightBasis(?)"; +} + +// Explicit edge or source weights, converted once and shared by policies built from the model. +// Scan all shape defects, then integer types, then int64 storage overflow (ValueError). +// The policy checks graph-sized length and the range [0, 2^62); only the diagnostic name differs. +std::shared_ptr> weights_of(const py::object& weights, const char* name) { + if (weights.is_none()) return {}; + (void)sequence_length(weights, name); + const auto w = wide_integers(weights, name); + for (std::size_t e = 0; e < w.size(); ++e) { + if (w.sign(e) != 0) throw py::value_error(std::string(name) + "[" + std::to_string(e) + "] must be an integer in [0, 2^62)"); + } + return std::make_shared>(w.narrow()); +} + +// A CostStretch factor as its one exact fraction, converted in this order: a bool (Python or +// numpy.bool_) raises TypeError; a fractions.Fraction is taken as it is; a Python float or a NumPy +// floating value is Fraction(str(f)), the shortest decimal its own type prints (a NaN or an infinity +// raises ValueError); an integer (an object with __index__) is n/1; anything else raises TypeError. +// Then the reduced components: a numerator in [-2^63, 2^63), a denominator in [1, 2^63). +CostStretch cost_stretch_of(const py::object& factor) { + const auto type_error = [&] { + return py::type_error("CostStretch factor must be a Fraction, a float or an integer (got " + + std::string(py::str(py::type::of(factor).attr("__name__"))) + ")"); + }; + if (is_bool(factor)) throw type_error(); + const py::object fraction_type = py::module_::import("fractions").attr("Fraction"); + py::object fraction; + if (py::isinstance(factor, fraction_type)) { + fraction = factor; + } else if (PyFloat_Check(factor.ptr()) || has_type(factor.ptr(), numpy_number_types().second)) { + const double x = PyFloat_AsDouble(factor.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + if (!std::isfinite(x)) throw py::value_error("CostStretch factor must be finite (got " + std::string(py::str(factor)) + ")"); + fraction = fraction_type(py::str(factor)); + } else if (PyIndex_Check(factor.ptr())) { + const auto n = py::reinterpret_steal(PyNumber_Index(factor.ptr())); + if (!n) throw py::error_already_set(); + fraction = fraction_type(n); + } else { + throw type_error(); + } + const auto component = [&](const char* name, std::int64_t lo, const char* text) { + const py::object v = fraction.attr(name); + int overflow = 0; + const long long x = PyLong_AsLongLongAndOverflow(v.ptr(), &overflow); + if (x == -1 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + if (overflow != 0 || x < lo) throw py::value_error(text); + return static_cast(x); + }; + const auto n = component("numerator", std::numeric_limits::min(), + "CostStretch fraction numerator must be in [-2^63, 2^63)"); + const auto d = component("denominator", 1, "CostStretch fraction denominator must be in [1, 2^63)"); + return CostStretch(n, d); +} + +} // namespace + +void register_routing(py::module_& m) { + py::class_(m, "AbsoluteCost", "Cost ceiling: an absolute, inclusive path-cost budget (cost >= 0).") + .def(py::init([](const py::object& cost) { + return AbsoluteCost {integer_value(cost, std::numeric_limits::min(), std::numeric_limits::max(), + "AbsoluteCost cost")}; + }), + py::arg("cost"), "AbsoluteCost(cost): a 64-bit integer budget in cost units (checked >= 0 where it is used)") + .def_property_readonly("cost", int_reader(&AbsoluteCost::cost), "the budget: a path qualifies when its cost is at most this") + .def("__eq__", [](const AbsoluteCost& a, const py::object& b) { return py::isinstance(b) && a == b.cast(); }) + .def("__hash__", [](const AbsoluteCost& a) { return hash_int(objects_tuple({str_object("AbsoluteCost"), int_object(a.cost)})); }) + .def("__repr__", [](const AbsoluteCost& a) { return "AbsoluteCost(" + std::to_string(a.cost) + ")"; }); + + py::class_(m, "CostStretch", + "Cost ceiling: a path qualifies when cost * d <= n * base, exactly, where n/d is the factor as\n" + "one reduced fraction and base the static shortest cost under the Eligibility (T1: the smallest\n" + "over all roster heads).") + .def(py::init([](const py::object& factor) { return cost_stretch_of(factor); }), py::arg("factor"), + "CostStretch(factor): in this order, a bool (or numpy.bool_) raises TypeError; a fractions.Fraction\n" + "is taken as it is; a Python float or any NumPy floating value is Fraction(str(f)), the shortest\n" + "decimal its own type prints (1.2, np.float64(1.2) and np.float32(1.2) are all 6/5), and a NaN or\n" + "an infinity raises ValueError; an integer, NumPy integer or any __index__ object is n/1 exactly;\n" + "anything else raises TypeError. Then a reduced numerator outside [-2^63, 2^63) or denominator\n" + "outside [1, 2^63) raises ValueError. A factor below 1 is valid here and refused where the\n" + "ceiling is used.") + .def_property_readonly( + "factor", + [](const CostStretch& a) { + return fraction_object(py::module_::import("fractions").attr("Fraction"), a.numerator(), a.denominator()); + }, + "the factor as a fractions.Fraction (built on each access)") + .def("__eq__", [](const CostStretch& a, const py::object& b) { return py::isinstance(b) && a == b.cast(); }) + .def("__hash__", + [](const CostStretch& a) { + return hash_int(fraction_object(py::module_::import("fractions").attr("Fraction"), a.numerator(), a.denominator())); + }) + .def("__repr__", [](const CostStretch& a) { + return "CostStretch(Fraction(" + std::to_string(a.numerator()) + ", " + std::to_string(a.denominator()) + "))"; + }); + + py::class_(m, "TunnelCount", + "The tunnel count of T1 (DistributedTE) and T2, per head: per_head in [1, 10000] (checked when a\n" + "policy or an evaluator uses it). up_to=True (T2 only; T1 rejects it): one common count k in\n" + "1..per_head for every head: the smallest common count whose program carries the live heads'\n" + "design shares; if none does, the largest live-share score, then the smallest count. It does not\n" + "promise whole-demand lossless delivery.") + .def(py::init([](const py::object& count, StrictBool up_to) { + return TunnelCount {integer_value(count, std::numeric_limits::min(), + std::numeric_limits::max(), "TunnelCount per_head"), + up_to}; + }), + py::arg("per_head"), py::kw_only(), py::arg("up_to") = StrictBool {false}, + "TunnelCount(per_head, *, up_to=False): a 64-bit integer per_head (a bool, a float or None raises\n" + "TypeError and an integer outside 64 bits ValueError here; [1, 10000] is checked where it is\n" + "used) and a bool") + .def_property_readonly("per_head", int_reader(&TunnelCount::per_head), "the tunnels per head (the largest count for up_to)") + .def_readonly("up_to", &TunnelCount::up_to, "True: the smallest count up to per_head whose program carries the live heads' design shares (else the\n" + "best live-share score, then the smallest); T2 only") + .def("__eq__", [](const TunnelCount& a, const py::object& b) { return py::isinstance(b) && a == b.cast(); }) + .def("__hash__", [](const TunnelCount& a) { + return hash_int(objects_tuple({str_object("TunnelCount"), int_object(a.per_head), py::bool_(a.up_to)})); + }) + .def("__repr__", [](const TunnelCount& a) { + return "TunnelCount(" + std::to_string(a.per_head) + (a.up_to ? ", up_to=True)" : ")"); + }); + + py::class_(m, "ShortestPaths", + "Routing like IGP forwarding (OSPF, IS-IS routers) on each roster source's static shortest-path\n" + "DAG (read from the ShortestPathCache; the route choice is applied when the program is prepared:\n" + "a positive place of an epoch without one, or any replace or evaluate), fixed for the epoch;\n" + "one record per roster source. EQUAL (ECMP, the default) and WEIGHTED\n" + "(WCMP) pool the sources (source allocation EQUAL): where their routes meet, a router splits\n" + "their traffic once. OPTIMIZED is a controller on the union of those routes (source allocation\n" + "OPTIMIZED by default, or EQUAL). An edge of capacity 0 stays in the DAG: EQUAL still sends it\n" + "its share, which it cannot carry, and WEIGHTED gives it weight 0.") + .def(py::init([](RouteAllocation allocation, std::optional source_allocation, + std::optional weight_basis, const py::object& edge_weights, RouteTieBreak tie_break, + StrictBool multipath, StrictBool parallel_edges) { + ShortestPaths sp; + sp.allocation = allocation; + sp.source_allocation = source_allocation; + sp.weight_basis = weight_basis; + sp.edge_weights = weights_of(edge_weights, "edge_weights"); + sp.tie_break = tie_break; + sp.multipath = multipath; + sp.parallel_edges = parallel_edges; + return sp; + }), + py::kw_only(), py::arg_v("allocation", RouteAllocation::Equal, "RouteAllocation.EQUAL"), + py::arg("source_allocation") = py::none(), py::arg("weight_basis") = py::none(), + py::arg("edge_weights") = py::none(), py::arg_v("tie_break", RouteTieBreak::FewerHops, "RouteTieBreak.FEWER_HOPS"), + py::arg("multipath") = StrictBool {true}, py::arg("parallel_edges") = StrictBool {true}, + "ShortestPaths(*, allocation=EQUAL, source_allocation=None, weight_basis=None, edge_weights=None,\n" + "tie_break=FEWER_HOPS, multipath=True, parallel_edges=True): ShortestPaths() is ECMP.\n" + "source_allocation None resolves to EQUAL for EQUAL and WEIGHTED (the only allowed value) and to\n" + "OPTIMIZED for OPTIMIZED; weight_basis (WEIGHTED only; None: CAPACITY) and edge_weights (one\n" + "integer per edge, given exactly when weight_basis=EXPLICIT; one array shared by every policy\n" + "built from this model). The combination is checked when a policy or an evaluator uses the model\n" + "(ValueError, in the order of the model matrix). Here, a value of the wrong type (a multipath or parallel_edges that\n" + "is not a bool, an enumeration of another class, an edge_weights that is not a 1-D sequence of\n" + "integers or has a bool or a float entry) raises TypeError, and an edge_weights entry beyond\n" + "int64 ValueError.") + .def_property_readonly("allocation", field_object(&ShortestPaths::allocation), "the RouteAllocation: EQUAL, WEIGHTED or OPTIMIZED") + .def_property_readonly("source_allocation", optional_field(&ShortestPaths::source_allocation), + "the SourceAllocation, or None (a policy's config: resolved to the allocation's default)") + .def_property_readonly("weight_basis", optional_field(&ShortestPaths::weight_basis), + "the WeightBasis, or None (a policy's config: CAPACITY resolved for WEIGHTED)") + .def_property_readonly( + "edge_weights", + [](const ShortestPaths& sp) -> py::object { + if (!sp.edge_weights) return py::none(); + return amounts_tuple(*sp.edge_weights); + }, + "the explicit weights (one int per edge, a tuple), or None") + .def_property_readonly("tie_break", field_object(&ShortestPaths::tie_break), "the RouteTieBreak of equal-cost choices") + .def_readonly("multipath", &ShortestPaths::multipath, + "True: every equal-cost route; False: one canonical path per roster source toward the destination set") + .def_readonly("parallel_edges", &ShortestPaths::parallel_edges, + "True: every eligible parallel edge; False: one per (u, v) pair by the tie rule") + .def("__repr__", [](const ShortestPaths& sp) { + return "ShortestPaths(allocation=" + std::string(allocation_repr(sp.allocation)) + ", source_allocation=" + + (sp.source_allocation ? source_allocation_repr(*sp.source_allocation) : "None") + + ", weight_basis=" + (sp.weight_basis ? weight_basis_repr(*sp.weight_basis) : "None") + + ", edge_weights=" + (sp.edge_weights ? "(...)" : "None") + + ", tie_break=" + tie_break_repr(sp.tie_break) + ", multipath=" + (sp.multipath ? "True" : "False") + + ", parallel_edges=" + (sp.parallel_edges ? "True" : "False") + ")"; + }); + + py::class_( + m, "DistributedTE", + "T1: every head sets up its own strict tunnels by CSPF, no controller (like RSVP-TE head ends).\n" + "The design volume D is split equally over the roster heads and each head's share equally over its\n" + "tunnel_count.per_head tunnels; in round order (round j, heads ascending: entry_id j*H + i) each tunnel\n" + "keeps the allowed links whose unreserved capacity (the ledger's, minus the holds of every other\n" + "owner and of this setup's earlier tunnels) is at least max(request, min_unreserved_bandwidth) (a\n" + "request of 0 with a minimum of 0 has no filter) and takes the shortest path in the tie_break's\n" + "single-path order; it comes up holding exactly its request on every link (UP), or stays down\n" + "holding nothing: INSUFFICIENT_BANDWIDTH (no path passes the filter) or COST_CEILING_EXCEEDED (the cheapest exceeds\n" + "cost_ceiling: AbsoluteCost is inclusive; CostStretch(f) is f times the smallest static shortest cost over\n" + "all roster heads). Traffic enters equally at the heads and splits equally over each head's up\n" + "tunnels; a head with no tunnel up loses its share. Holds never police traffic.") + .def(py::init([](const TunnelCount& tunnels, const py::object& cost_ceiling, const py::object& min_unreserved, + RouteTieBreak tie_break, StrictBool parallel_edges) { + DistributedTE ca; + ca.tunnel_count = tunnels; + ca.tie_break = tie_break; + ca.parallel_edges = parallel_edges; + ca.cost_ceiling = cost_ceiling_of(cost_ceiling); + ca.min_unreserved_bandwidth = integer_value(min_unreserved, std::numeric_limits::min(), + std::numeric_limits::max(), "min_unreserved_bandwidth"); + return ca; + }), + py::arg("tunnel_count"), py::kw_only(), py::arg("cost_ceiling") = py::none(), + py::arg("min_unreserved_bandwidth") = 0, + py::arg_v("tie_break", RouteTieBreak::FewerHops, "RouteTieBreak.FEWER_HOPS"), + py::arg("parallel_edges") = StrictBool {true}, + "DistributedTE(tunnel_count, *, cost_ceiling=None, min_unreserved_bandwidth=0, tie_break=FEWER_HOPS,\n" + "parallel_edges=True): the fields are checked when a policy is built with them (the count in\n" + "[1, 10000] and exact, the ceiling, min_unreserved_bandwidth in [0, 2^62) units). tie_break and\n" + "parallel_edges are the CSPF choice, scored on unreserved bandwidth. Here, a\n" + "min_unreserved_bandwidth outside 64 bits raises ValueError; one of the wrong type, a tunnel_count\n" + "that is not a TunnelCount, a ceiling of the wrong class, a parallel_edges that is not a bool and an\n" + "unknown keyword raise TypeError.") + .def_property_readonly( + "tunnel_count", [](const DistributedTE& ca) { return typed_object(ca.tunnel_count); }, + "the tunnel count per head (a TunnelCount)") + .def_property_readonly( + "cost_ceiling", + [](const DistributedTE& ca) -> py::object { + if (!ca.cost_ceiling.has_value()) return py::none(); + return std::visit([](const auto& x) { return object_of(x); }, *ca.cost_ceiling); + }, + "the inclusive route cost ceiling (AbsoluteCost or CostStretch); None: no ceiling") + .def_property_readonly("min_unreserved_bandwidth", int_reader(&DistributedTE::min_unreserved_bandwidth), + "units in [0, 2^62): a search minimum only (each attempt keeps the links with at least\n" + "max(request, this) unreserved units), never a reservation") + .def_property_readonly("tie_break", field_object(&DistributedTE::tie_break), "the RouteTieBreak of the CSPF choice") + .def_readonly("parallel_edges", &DistributedTE::parallel_edges, + "True: every eligible parallel edge; False: one per (u, v) pair by the tie rule") + .def("__repr__", [](const DistributedTE& ca) { + std::string ceiling = "None"; + if (ca.cost_ceiling.has_value()) { + ceiling = std::visit([](const auto& x) { return own_repr(typed_object(x)); }, *ca.cost_ceiling); + } + return "DistributedTE(TunnelCount(" + std::to_string(ca.tunnel_count.per_head) + (ca.tunnel_count.up_to ? ", up_to=True)" : ")") + + ", cost_ceiling=" + ceiling + ", min_unreserved_bandwidth=" + std::to_string(ca.min_unreserved_bandwidth) + + ", tie_break=" + std::string(tie_break_repr(ca.tie_break)) + + ", parallel_edges=" + (ca.parallel_edges ? "True" : "False") + ")"; + }); + + py::class_(m, "CentralizedTE", + "Routing: an ideal central controller that sees the whole network, or only a support DAG. It takes\n" + "no route choice, and its source_allocation is required.\n" + "Without tunnel_count: a positive place re-solves the minimum-cost flow of what the policy\n" + "already delivers plus this call's request (delivered() + offer; what an earlier call could not\n" + "place is not requested again); evaluate and replace solve their whole offer afresh, and forward\n" + "sends its offer over the installed paths without a solve. A solve runs over the eligible graph,\n" + "or exactly over `support` (a\n" + "PredecessorDAG of the policy's graph, trimmed to the walks from the sources to the allowed\n" + "destinations under the Eligibility; every other edge carries nothing), on the free capacity\n" + "plus its own allocation: it may move its own traffic, never another's. OPTIMIZED: the controller\n" + "chooses how much enters at each source. EQUAL: every roster source sends its round-robin share\n" + "of the largest total whose shares all fit. WEIGHTED: highest-averages shares of the declared\n" + "source_weights; zero weights receive nothing, and all-zero roster weights end NO_ACTIVE_WEIGHT\n" + "on a positive live call. The program is the solve cut into paths per source, each a tunnel\n" + "weighted by its amount; it keeps the declared weights for weighted sources and the solved\n" + "supplies for optimized ones. A positive call on a network beyond the flow core's int32\n" + "numbering raises ValueError (see FlowPolicy.place).\n" + "With tunnel_count=TunnelCount(N) (T2; with support, every step uses only the support's edges\n" + "allowed by the policy's mask): a controller (like a PCE) lays out, for every live head (one\n" + "reaching a destination over links with free capacity >= 1; the other heads are dead and lose\n" + "their share), N single-path tunnels with one common whole-unit planned bandwidth: first as\n" + "large as possible (U), then for a finite design no larger than needed (u), then the cheapest\n" + "routes, then with prefer_headroom=True the routes that leave the most spare capacity, then one\n" + "fixed unit-by-unit tie. TunnelCount(L, up_to=True): the smallest common count k <= L whose\n" + "program carries the live heads' design shares (else the largest live-share score, then the\n" + "smallest count); no promise of whole-demand lossless delivery. Traffic splits equally over the\n" + "heads and over each head's tunnels; planned bandwidth is never held.") + .def(py::init([](SourceAllocation source_allocation, const py::object& source_weights, const py::object& support, const py::object& tunnels, + StrictBool prefer_headroom) { + CentralizedTE gl(source_allocation); + gl.source_weights = weights_of(source_weights, "source_weights"); + gl.prefer_headroom = prefer_headroom; + if (!support.is_none()) { + if (!py::isinstance(support)) throw py::type_error("CentralizedTE support must be a PredecessorDAG or None"); + gl.support = support.cast>(); + } + if (!tunnels.is_none()) { + if (!py::isinstance(tunnels)) throw py::type_error("CentralizedTE tunnel_count must be a TunnelCount or None"); + gl.tunnel_count = tunnels.cast(); + } + return gl; + }), + py::kw_only(), py::arg("source_allocation"), py::arg("source_weights") = py::none(), py::arg("support") = py::none(), + py::arg("tunnel_count") = py::none(), py::arg("prefer_headroom") = StrictBool {false}, + "CentralizedTE(*, source_allocation, source_weights=None, support=None, tunnel_count=None, prefer_headroom=False):\n" + "source_allocation is required (EQUAL, OPTIMIZED or WEIGHTED; None or a missing argument raises TypeError).\n" + "WEIGHTED requires source_weights and no tunnel_count: one integer per graph node in [0, 2^62),\n" + "copied once and exposed as an immutable tuple. Other source allocations refuse source_weights.\n" + "support a PredecessorDAG of the policy's graph (checked when a policy is built with it: `support\n" + "belongs to another graph`) or None for every eligible edge; tunnel_count a TunnelCount (T2;\n" + "the count in [1, 10000] is checked when a policy is built with it) or None; prefer_headroom (T2\n" + "only; True without tunnel_count raises `prefer_headroom requires tunnel_count` when a policy is\n" + "built with it): the headroom penalty as T2's objective after route cost") + .def_property_readonly("source_allocation", field_object(&CentralizedTE::source_allocation), "the SourceAllocation: EQUAL, OPTIMIZED or WEIGHTED") + .def_property_readonly("source_weights", [](const CentralizedTE& gl) -> py::object { + if (!gl.source_weights) return py::none(); + return amounts_tuple(*gl.source_weights); + }, "declared node-indexed source weights as an immutable tuple, or None") + .def_property_readonly( + "support", + [](const CentralizedTE& gl) { + return optional_shared(gl.support); + }, + "the supplied support (a PredecessorDAG), or None") + .def_property_readonly( + "tunnel_count", [](const CentralizedTE& gl) { return optional_of(gl.tunnel_count); }, + "T2's tunnel count per head (a TunnelCount), or None") + .def_readonly("prefer_headroom", &CentralizedTE::prefer_headroom, + "True: T2 minimizes the headroom penalty after route cost (T2 only)") + .def("__repr__", [](const CentralizedTE& gl) { + // The source allocation and every other field that is set. + std::string out; + const auto add = [&](const std::string& part) { out += (out.empty() ? "" : ", ") + part; }; + add("source_allocation=" + std::string(source_allocation_repr(gl.source_allocation))); + if (gl.source_weights) add("source_weights=..."); + if (gl.support) add("support=..."); + if (gl.tunnel_count.has_value()) { + add("tunnel_count=TunnelCount(" + std::to_string(gl.tunnel_count->per_head) + (gl.tunnel_count->up_to ? ", up_to=True)" : ")")); + } + if (gl.prefer_headroom) add("prefer_headroom=True"); + return "CentralizedTE(" + out + ")"; + }); + + py::class_(m, "Pinned", + "Routing: forwarding over a given program, computing nothing (the program's endpoints must be\n" + "the policy's). The policy's Eligibility repairs failures: a tunnel touching an excluded element is\n" + "FAILED, a split route loses its excluded members, and the roster is rebuilt from the program's\n" + "basis (a head that no longer reaches a destination leaves the equal pool). black_holes: the nodes\n" + "and edges this Eligibility excludes are silent failures; the program is forwarded unchanged and\n" + "traffic reaching them is dropped there (stale forwarding; None: no black holes). One record per\n" + "program entry.") + .def(py::init([](const std::shared_ptr& program, const std::shared_ptr& black_holes) { + if (!program) throw py::type_error("Pinned program must be a ForwardingProgram"); + Pinned p; + p.program = program; + p.black_holes = black_holes; + return p; + }), + py::arg("program"), py::kw_only(), py::arg("black_holes") = nullptr, + "Pinned(program, *, black_holes=None): a ForwardingProgram and an Eligibility of the same graph (both\n" + "kept by reference; checked when a policy is built with them)") + .def_property_readonly( + "program", [](const Pinned& p) { return typed_shared(p.program); }, + "the forwarded program (shared, immutable)") + .def_property_readonly( + "black_holes", + [](const Pinned& p) { + return optional_shared(p.black_holes); + }, + "the black holes (an Eligibility), or None") + .def("__repr__", [](const Pinned& p) { + return "Pinned(tunnels=" + std::to_string(p.program->tunnels().size()) + ")"; + }); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/search.cpp b/bindings/python/search.cpp new file mode 100644 index 0000000..5f5eabb --- /dev/null +++ b/bindings/python/search.cpp @@ -0,0 +1,206 @@ +// spf with SpfOptions and SpfResult, the public search, and UNREACHABLE. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/detail/endpoints.hpp" +#include "netgraph/core/detail/enum_domain.hpp" +#include "netgraph/core/detail/search_detail.hpp" +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/options.hpp" +#include "netgraph/core/route_structures.hpp" +#include "netgraph/core/search.hpp" +#include "netgraph/core/types.hpp" +#include "integers.hpp" +#include "strict_bool.hpp" +#include "publication.hpp" +#include "values.hpp" +#include "registration.hpp" +#include "native_call.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +// The Python form of SpfOptions: the arguments as given (targets, residual and min_residual are +// validated at the call, in spf's validation order, on their unnarrowed values); the Eligibility is +// held by reference count. +struct PySpfOptions { + Direction direction; + RouteTieBreak tie_break; + bool multipath; + bool parallel_edges; + py::object targets; + py::object residual; + py::object min_residual; + std::shared_ptr eligibility; +}; + +// The Python form of SpfResult: every field already a Python object (made by the call through +// the checked allocations), so reading one allocates nothing. +struct PySpfResult { + py::object costs; // a read-only int64 array (full searches) or None + py::object dag; // PredecessorDAG + py::object target_cost; // an int, or None (no target reached, or a full search) +}; + +} // namespace + +void register_search(py::module_& m) { + m.attr("UNREACHABLE") = py::int_(kUnreachable); + + py::class_( + m, "SpfOptions", + "The options of spf: the direction; targets (one NodeId or a sequence; None: a full search)\n" + "stops the search at the nearest targets (anycast: any nearest target will do); tie_break\n" + "(RouteTieBreak) orders equal-cost choices; multipath (True: every equal-cost walk as a DAG,\n" + "False: the one canonical path, targets required); parallel_edges (True: every eligible parallel\n" + "edge, False: one per (u, v) pair by the tie rule); a HIGHER_* tie_break with multipath and\n" + "parallel_edges both True is rejected by spf; residual (int64 units per EdgeId, each in [0,\n" + "edge_capacity_units]) makes the search skip edges with fewer than min_residual residual units\n" + "(an integer >= 1; without a residual it has no effect) and supplies the availability: it is the\n" + "only capacity threshold, and without it an edge of capacity 0 is searched like any other;\n" + "eligibility (None: everything allowed). A direction outside its enumeration raises ValueError\n" + "here, a multipath or parallel_edges that is not a bool TypeError; the other fields are validated\n" + "by spf, in the order its docstring gives.") + .def(py::init([](Direction direction, const py::object& targets, RouteTieBreak tie_break, StrictBool multipath, + StrictBool parallel_edges, const py::object& residual, const py::object& min_residual, + const std::shared_ptr& eligibility) { + detail::check_member(direction, "direction"); + return PySpfOptions {direction, tie_break, multipath, parallel_edges, targets, residual, min_residual, eligibility}; + }), + py::arg("direction"), py::kw_only(), py::arg("targets") = py::none(), + py::arg_v("tie_break", RouteTieBreak::FewerHops, "RouteTieBreak.FEWER_HOPS"), + py::arg("multipath") = StrictBool {true}, py::arg("parallel_edges") = StrictBool {true}, + py::arg("residual") = py::none(), py::arg("min_residual") = 1, py::arg("eligibility") = nullptr) + .def_property_readonly("direction", field_object(&PySpfOptions::direction), "FORWARD or REVERSE") + .def_property_readonly("tie_break", field_object(&PySpfOptions::tie_break), "the RouteTieBreak of equal-cost choices") + .def_readonly("multipath", &PySpfOptions::multipath, + "True: every equal-cost walk (a DAG); False: the canonical single path") + .def_readonly("parallel_edges", &PySpfOptions::parallel_edges, + "True: every eligible parallel edge; False: one per (u, v) pair by the tie rule") + .def_readonly("targets", &PySpfOptions::targets, "the targets as given (validated by spf); None: a full search") + .def_readonly("residual", &PySpfOptions::residual, + "the residual units per EdgeId as given (validated by spf); None: no capacity filter;\n" + "the tie scores read static capacity") + .def_readonly("min_residual", &PySpfOptions::min_residual, + "the gating threshold in units as given (validated by spf: an integer >= 1)") + .def_property_readonly("eligibility", shared_field(&PySpfOptions::eligibility), "the static Eligibility; None: everything allowed"); + + py::class_(m, "SpfResult", + "The result of spf: costs (full searches: a read-only int64 array, UNREACHABLE for the nodes not\n" + "reached; None with targets), dag (PredecessorDAG of the tight edges, the edges on some shortest walk,\n" + "one per (u, v) pair with parallel_edges = False, each in its own direction; with targets only the\n" + "walks between the roots and a nearest target, for REVERSE from the target to a root; for\n" + "multipath = False the canonical single path) and\n" + "target_cost (with targets: the nearest targets' cost, None when no target is reached; None for a\n" + "full search).") + .def_property_readonly( + "costs", [](const PySpfResult& r) { return r.costs ? r.costs : py::object(py::none()); }, + "full search: each node's distance from the roots for FORWARD, or to the roots for REVERSE\n" + "(read-only int64[N], UNREACHABLE for nodes not reached); None with targets") + .def_property_readonly( + "dag", [](const PySpfResult& r) { return r.dag; }, + "the PredecessorDAG of the result, in the edges' original orientation") + .def_property_readonly( + "target_cost", [](const PySpfResult& r) { return r.target_cost ? r.target_cost : py::object(py::none()); }, + "with targets: the cost of the nearest targets, None when no target is reached; None for a\n" + "full search") + .def("__repr__", [](const PySpfResult& r) { + std::string tc = r.target_cost && !r.target_cost.is_none() ? std::string(py::str(r.target_cost)) : "None"; + return "SpfResult(costs=" + std::string(r.costs ? "array" : "None") + + ", dag_entries=" + std::to_string(r.dag.cast().via_edges().size()) + + ", target_cost=" + tc + ")"; + }); + + m.def( + "spf", + [](const std::shared_ptr& graph, const py::object& roots, const py::object& opts) { + if (!graph) throw py::type_error("graph must be a Graph"); + const auto o = options_of(opts, "SpfOptions"); + const auto n = graph->num_nodes(); + // spf's validation order on unnarrowed values (Core repeats every check): the enumerations, + // the roots, the targets, their overlap, a single path without targets, the ineffective tie + // rule, the residual, then min_residual and a foreign Eligibility. + detail::check_member(o.direction, "direction"); + detail::check_member(o.tie_break, "tie_break"); + const auto r = node_ids(roots, n, "roots"); + std::optional> t; + if (!o.targets.is_none()) { + t = node_ids(o.targets, n, "targets"); + detail::check_disjoint(detail::node_set(r, n, "roots"), detail::node_set(*t, n, "targets"), "roots", + "targets"); + } else if (!o.multipath) { + throw py::value_error("multipath = false requires targets (the single path joins a root to a target)"); + } + if (detail::RouteChoice {o.tie_break, o.multipath, o.parallel_edges}.tie_is_ineffective()) { + throw py::value_error("spf tie_break requires multipath=False or parallel_edges=False"); + } + std::optional> res; + if (!o.residual.is_none()) res = edge_units(o.residual, *graph, "residual"); + const auto minr = python_int(o.min_residual, "min_residual"); + int overflow = 0; + const long long mv = PyLong_AsLongLongAndOverflow(minr.ptr(), &overflow); + if (overflow < 0 || (overflow == 0 && mv < 1)) { + throw py::value_error("min_residual must be at least 1 unit (got " + std::string(py::str(minr)) + ")"); + } + SpfOptions c(o.direction); + c.tie_break = o.tie_break; + c.multipath = o.multipath; + c.parallel_edges = o.parallel_edges; + c.targets = std::move(t); + if (res.has_value()) c.residual = std::span(*res); + // Above INT64_MAX it gates every residual edge exactly as INT64_MAX does (residuals < 2^62). + c.min_residual = overflow != 0 ? std::numeric_limits::max() : static_cast(mv); + c.eligibility = o.eligibility.get(); + if (c.eligibility && c.eligibility->graph_uid() != graph->uid()) { + throw py::value_error("foreign Eligibility: built for another graph"); + } + auto result = native_call([&] { return spf(*graph, r, c); }); + // The result's Python objects, each through a checked allocation: a failure raises. + auto [obj, slot] = new_object(); + if (result.costs.has_value()) { + py::array_t costs(static_cast(result.costs->size())); + std::copy(result.costs->begin(), result.costs->end(), costs.mutable_data()); + py::detail::array_proxy(costs.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + slot->costs = std::move(costs); + } + slot->dag = object_from(std::move(result.dag)); + if (result.target_cost.has_value() && *result.target_cost != kUnreachable) { + slot->target_cost = int_object(*result.target_cost); + } + return std::move(obj); + }, + py::arg("graph"), py::arg("roots"), py::arg("options"), + "Shortest paths from the roots for FORWARD, or to the roots for REVERSE (one NodeId or a sequence;\n" + "every root at distance 0), over the eligible edges (only those with residual >= min_residual when\n" + "a residual is given; an edge of capacity 0 is searched like any other) -> SpfResult. Without\n" + "targets every reachable node gets its distance: costs and the full DAG of tight edges (the edges\n" + "on some shortest walk; one per (u, v) pair with parallel_edges = False). With targets the search\n" + "stops at the nearest targets' cost:\n" + "target_cost and the DAG of the walks between the roots and a nearest target (for REVERSE, from\n" + "the target to a root, in the edges' own direction);\n" + "with multipath = False, instead the canonical single path, the walk minimising (cost, -bottleneck\n" + "under a HIGHER_* tie_break, hops, EdgeId sequence). Validated in this order before any work\n" + "(ValueError): the roots (non-empty, ids in range), the targets (the same), a node in both sets,\n" + "multipath = False without targets, a HIGHER_* tie_break with multipath and parallel_edges both\n" + "True ('spf tie_break requires multipath=False or parallel_edges=False'), the residual (length,\n" + "then every entry in [0, edge_capacity_units]),\n" + "min_residual >= 1, a foreign Eligibility. Excluded roots and targets are skipped; all roots\n" + "excluded gives an empty result."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/shortest_path_cache.cpp b/bindings/python/shortest_path_cache.cpp new file mode 100644 index 0000000..a4da871 --- /dev/null +++ b/bindings/python/shortest_path_cache.cpp @@ -0,0 +1,62 @@ +// ShortestPathCacheStats, ShortestPathCacheMiss and ShortestPathCache: the cache of static shortest-path DAGs. +#include +#include +#include + +#include + +#include "netgraph/core/shortest_path_cache.hpp" +#include "netgraph/core/graph.hpp" +#include "registration.hpp" +#include "native_call.hpp" +#include "values.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +void register_shortest_path_cache(py::module_& m) { + py::class_(m, "ShortestPathCacheStats", + "ShortestPathCache statistics (atomic per field, not a consistent snapshot): hits, misses,\n" + "entries, bytes (the entries' retained payload), live_values (values alive anywhere).") + .def_property_readonly("hits", int_reader(&ShortestPathCache::Stats::hits), "lookups answered from an entry (they persist across clear())") + .def_property_readonly("misses", int_reader(&ShortestPathCache::Stats::misses), + "lookups without an entry: a building cache searched, a frozen one raised ShortestPathCacheMiss") + .def_property_readonly("entries", int_reader(&ShortestPathCache::Stats::entries), "the entries held") + .def_property_readonly("bytes", int_reader(&ShortestPathCache::Stats::bytes), "the retained payload of the entries (their DAGs' stored arrays)") + .def_property_readonly("live_values", int_reader(&ShortestPathCache::Stats::live_values), "the cached values alive anywhere, entries included"); + + py::register_exception(m, "ShortestPathCacheMiss", PyExc_Exception); + m.attr("ShortestPathCacheMiss").attr("__doc__") = + "A lookup in a frozen ShortestPathCache whose key is absent: a dedicated exception (not a\n" + "RuntimeError or ValueError); the evaluators propagate it unchanged."; + + py::class_>( + m, "ShortestPathCache", + "A cache of static shortest-path DAGs (from costs and the Eligibility only) for one graph, keyed by\n" + "(the root set, the direction, the Eligibility uid). It stores the full DAG and applies each\n" + "caller's route choice afterwards, so the route choice is never cached. ShortestPaths policies and the\n" + "evaluators read it.\n" + "Building mode (a new cache): one exclusive lease per call; a miss searches and inserts;\n" + "clear() empties it (the hit and miss counters persist), and values already handed out stay valid.\n" + "Frozen mode (freeze()): read-only and shareable by every worker; a miss counts one miss and raises\n" + "ShortestPathCacheMiss; clear() raises ValueError; thaw() waits for active calls, then returns to building.\n" + "Lifecycle calls take exclusive FIFO admission and cannot be overtaken by later calls.\n" + "When a public call fails: a building cache passed to it keeps the complete entries and statistics\n" + "of the lookups that succeeded before the failure (never a partial entry; a kept entry never\n" + "changes a result); a frozen cache's statistics stay unchanged, except one miss when the call\n" + "raised ShortestPathCacheMiss. A frozen cache counts its hits only when the call succeeds, Python result\n" + "included.") + .def(py::init([](const std::shared_ptr& graph) { + if (!graph) throw py::type_error("graph must be a Graph"); + return std::make_shared(*graph); + }), + py::arg("graph"), "An empty cache in building mode, bound to graph (by its uid).") + .def("graph_uid", &ShortestPathCache::graph_uid, "uid of the graph this cache is bound to") + .def("freeze", [](ShortestPathCache& cache) { native_call([&] { cache.freeze(); }); }, "wait for active calls and enter frozen mode (no effect when frozen)") + .def("thaw", [](ShortestPathCache& cache) { native_call([&] { cache.thaw(); }); }, "wait for active calls and return to building mode (no effect when building)") + .def("clear", [](ShortestPathCache& cache) { native_call([&] { cache.clear(); }); }, "empty a building cache; ValueError when frozen") + .def("stats", &ShortestPathCache::stats, "the current ShortestPathCacheStats (each field read atomically)"); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/strict_bool.hpp b/bindings/python/strict_bool.hpp new file mode 100644 index 0000000..1722bb5 --- /dev/null +++ b/bindings/python/strict_bool.hpp @@ -0,0 +1,70 @@ +/* + The one bool converter of every bool parameter: exactly a Python bool or a numpy.bool_, both by + type identity. pybind11's own bool caster in no-convert mode recognizes NumPy's bool by the name + of its type, so a class named "numpy.bool_" passes it; this caster compares the type object. + Also NumPy's number types and the check of an object's real type against them: isinstance would + honour an object's own __class__, which may claim any class. +*/ +#pragma once + +#include +#include + +#include + +namespace netgraph::core::bindings { + +// A bool parameter's value (its signature says bool). +struct StrictBool { + bool value {false}; + constexpr operator bool() const noexcept { return value; } // NOLINT(google-explicit-constructor) +}; + +// numpy.bool_'s type object, looked up once. +inline PyObject* numpy_bool_type() { + PYBIND11_CONSTINIT static pybind11::gil_safe_call_once_and_store storage; + return storage.call_once_and_store_result([] { return pybind11::module_::import("numpy").attr("bool_"); }).get_stored().ptr(); +} + +// numpy.integer and numpy.floating, looked up once (a lookup per call would import NumPy each time). +inline const std::pair& numpy_number_types() { + PYBIND11_CONSTINIT static pybind11::gil_safe_call_once_and_store> storage; + return storage + .call_once_and_store_result([] { + const auto np = pybind11::module_::import("numpy"); + return std::make_pair(pybind11::object(np.attr("integer")), pybind11::object(np.attr("floating"))); + }) + .get_stored(); +} + +// Whether o's real type (never its __class__) is `type` or a subclass of it. +inline bool has_type(PyObject* o, const pybind11::object& type) { + return PyType_IsSubtype(Py_TYPE(o), reinterpret_cast(type.ptr())) != 0; +} + +} // namespace netgraph::core::bindings + +namespace pybind11::detail { + +template <> +struct type_caster { + PYBIND11_TYPE_CASTER(netgraph::core::bindings::StrictBool, const_name("bool")); + + bool load(handle src, bool /*convert*/) { + if (src.ptr() == Py_True || src.ptr() == Py_False) { + value.value = src.ptr() == Py_True; + return true; + } + if (reinterpret_cast(Py_TYPE(src.ptr())) != netgraph::core::bindings::numpy_bool_type()) return false; + const int truth = PyObject_IsTrue(src.ptr()); + if (truth < 0) throw error_already_set(); + value.value = truth == 1; + return true; + } + + static handle cast(netgraph::core::bindings::StrictBool b, return_value_policy /*policy*/, handle /*parent*/) { + return handle(b.value ? Py_True : Py_False).inc_ref(); + } +}; + +} // namespace pybind11::detail diff --git a/bindings/python/strict_float.hpp b/bindings/python/strict_float.hpp new file mode 100644 index 0000000..e27e59b --- /dev/null +++ b/bindings/python/strict_float.hpp @@ -0,0 +1,67 @@ +/* + The float converter of the resolution parameters. It takes exactly a number: a float (a Python + float, read by its stored value, or a NumPy floating value, converted to double) or an integer (a Python int or a + NumPy integer, read exactly). Every other type is refused, and the call fails with pybind11's + TypeError for incompatible arguments: a bool (Python's or numpy.bool_, by type identity), a + fractions.Fraction, a decimal.Decimal, any object known only by __float__ or __index__. An + integer beyond the float range is a value: it becomes +inf or -inf, which the resolution's own + check refuses with its ValueError. An exception of the value's own conversion propagates as + raised. +*/ +#pragma once + +#include + +#include + +#include "strict_bool.hpp" + +namespace netgraph::core::bindings { + +// The double value of a parameter that takes concrete Python and NumPy numbers. +struct StrictFloat { + double value {0.0}; + constexpr operator double() const noexcept { return value; } // NOLINT(google-explicit-constructor) +}; + +} // namespace netgraph::core::bindings + +namespace pybind11::detail { + +template <> +struct type_caster { + // The input names exactly the concrete number types accepted by load(). + PYBIND11_TYPE_CASTER(netgraph::core::bindings::StrictFloat, io_name("int | float | numpy.integer | numpy.floating", "float")); + + bool load(handle src, bool convert) { + PyObject* o = src.ptr(); + if (PyBool_Check(o) || reinterpret_cast(Py_TYPE(o)) == netgraph::core::bindings::numpy_bool_type()) return false; + const auto& [np_integer, np_floating] = netgraph::core::bindings::numpy_number_types(); + if (PyFloat_Check(o) || netgraph::core::bindings::has_type(o, np_floating)) { + const double x = PyFloat_AsDouble(o); + if (x == -1.0 && PyErr_Occurred() != nullptr) throw error_already_set(); + value.value = x; + return true; + } + if (!convert || (PyLong_Check(o) == 0 && !netgraph::core::bindings::has_type(o, np_integer))) return false; + const object idx = reinterpret_steal(PyNumber_Index(o)); + if (!idx) throw error_already_set(); + const double x = PyLong_AsDouble(idx.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) { + if (PyErr_ExceptionMatches(PyExc_OverflowError) == 0) throw error_already_set(); + PyErr_Clear(); + const int sign = PyObject_RichCompareBool(idx.ptr(), int_(0).ptr(), Py_GT); + if (sign < 0) throw error_already_set(); + value.value = sign == 1 ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + return true; + } + value.value = x; + return true; + } + + static handle cast(netgraph::core::bindings::StrictFloat f, return_value_policy policy, handle parent) { + return type_caster::cast(f.value, policy, parent); + } +}; + +} // namespace pybind11::detail diff --git a/bindings/python/trace.hpp b/bindings/python/trace.hpp new file mode 100644 index 0000000..3e3f55e --- /dev/null +++ b/bindings/python/trace.hpp @@ -0,0 +1,55 @@ +/* + The trace of a traced placement as Python values (the private hooks only). +*/ +#pragma once + +#include +#include +#include + +#include "netgraph/core/detail/hooks_flag.hpp" + +#if NETGRAPH_CORE_PRIVATE_HOOKS +#include +#include + +#include "netgraph/core/detail/trace.hpp" +#include "values.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// A trace value as Python objects (map keys that are decimal integers become ints: the maps +// keyed by source NodeId), each made through a checked C API call: a failed allocation raises +// MemoryError (pybind11's int, str, list and dict constructors report it as RuntimeError). +inline py::object trace_obj(const detail::Trace& t) { + using K = detail::Trace::Kind; + switch (t.kind) { + case K::Null: return py::none(); + case K::Int: return checked(PyLong_FromLongLong(t.i)); + case K::Str: return checked(PyUnicode_FromStringAndSize(t.s.data(), static_cast(t.s.size()))); + case K::List: { + py::object out = checked(PyList_New(0)); + for (const auto& x : t.items) { + if (PyList_Append(out.ptr(), trace_obj(x).ptr()) != 0) throw py::error_already_set(); + } + return out; + } + case K::Map: { + py::object out = checked(PyDict_New()); + for (const auto& [k, x] : t.fields) { + const bool digits = !k.empty() && std::all_of(k.begin(), k.end(), [](char ch) { return ch >= '0' && ch <= '9'; }); + const py::object key = digits ? checked(PyLong_FromLongLong(std::stoll(k))) + : checked(PyUnicode_FromStringAndSize(k.data(), static_cast(k.size()))); + if (PyDict_SetItem(out.ptr(), key.ptr(), trace_obj(x).ptr()) != 0) throw py::error_already_set(); + } + return out; + } + } + return py::none(); +} + +} // namespace netgraph::core::bindings + +#endif diff --git a/bindings/python/units.cpp b/bindings/python/units.cpp new file mode 100644 index 0000000..358d2ea --- /dev/null +++ b/bindings/python/units.cpp @@ -0,0 +1,337 @@ +// to_units and as_float: physical values to integer units and back. +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/graph.hpp" +#include "netgraph/core/detail/unit_conversion.hpp" +#include "integers.hpp" +#include "strict_float.hpp" +#include "values.hpp" +#include "native_call.hpp" +#include "registration.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +namespace { + +constexpr const char* kNotPhysical = "to_units takes physical numbers (a float or an array of floats)"; +constexpr const char* kNotOneDimensional = "to_units takes a float or a 1-D array of floats"; + +std::string dtype_text(const py::array& a) { + return "to_units takes physical numbers (got dtype " + std::string(py::str(a.dtype())) + ")"; +} + +enum class Kind { Number, Wide, Bool, Other }; + +// An int's value as NumPy reads it, read from the int itself (a subclass's own methods are never +// called): through int64 or uint64, rounded to the nearest double (Number); beyond 64 bits NumPy +// holds it as an object (Wide: a value outside the domain). +Kind int_value(PyObject* o, double& value) { + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(o, &overflow); + if (v == -1 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + if (overflow == 0) { + value = static_cast(v); + return Kind::Number; + } + if (overflow > 0) { + const unsigned long long u = PyLong_AsUnsignedLongLong(o); + if (u != static_cast(-1) || PyErr_Occurred() == nullptr) { + value = static_cast(u); + return Kind::Number; + } + if (PyErr_ExceptionMatches(PyExc_OverflowError) == 0) throw py::error_already_set(); + PyErr_Clear(); + } + return Kind::Wide; +} + +// One scalar under the rule, in this order: a bool (Python's or numpy.bool_) is Bool; an int or a +// float, a subclass included, is a Number (Wide beyond 64 bits) read by its stored value (PyLong_* +// and PyFloat_AS_DOUBLE read the object itself, never its __int__, __float__ or sequence methods); +// a NumPy integer or floating scalar is a Number; anything else is Other. Every type is the +// object's real one (never its __class__). +Kind scalar_kind(const py::handle& x, double& value) { + PyObject* o = x.ptr(); + if (is_bool(x)) return Kind::Bool; + if (PyLong_Check(o) != 0) return int_value(o, value); + if (PyFloat_Check(o) != 0) { + value = PyFloat_AS_DOUBLE(o); + return Kind::Number; + } + const auto& [np_integer, np_floating] = numpy_number_types(); + if (has_type(o, np_integer)) { + const auto exact = checked(PyNumber_Index(o)); + return int_value(exact.ptr(), value); + } + if (has_type(o, np_floating)) { + value = PyFloat_AsDouble(o); + if (value == -1.0 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + return Kind::Number; + } + return Kind::Other; +} + +// Whether NumPy reads `o` without running code of the caller's: a str, bytes, bool, None, an exact +// int or float, an exact NumPy scalar or array (its type named "numpy.*"), or a list or tuple of +// those (nested at most 32 deep). +bool read_by_numpy_alone(PyObject* o, int depth = 0) { + if (PyUnicode_CheckExact(o) || PyBytes_CheckExact(o) || PyBool_Check(o) || o == Py_None || PyLong_CheckExact(o) || + PyFloat_CheckExact(o)) { + return true; + } + if (std::string_view(Py_TYPE(o)->tp_name).starts_with("numpy.")) return true; + if (depth >= 32 || (PyList_CheckExact(o) == 0 && PyTuple_CheckExact(o) == 0)) return false; + const Py_ssize_t n = PySequence_Fast_GET_SIZE(o); + PyObject** items = PySequence_Fast_ITEMS(o); + for (Py_ssize_t i = 0; i < n; ++i) { + if (!read_by_numpy_alone(items[i], depth + 1)) return false; + } + return true; +} + +// The refusal of an input that holds something other than numbers, always a TypeError, worded as +// NumPy reads it (the texts these inputs always had): a non-numeric dtype names it; a numeric +// reading of more than one dimension is the shape text; any other numeric reading the scalar text. +// An input NumPy would read through the caller's code (an object's own __array__, a subclass) is +// not given to NumPy: it holds an object, NumPy's `got dtype object`. +[[noreturn]] void refuse_physical(const py::handle& x) { + if (!read_by_numpy_alone(x.ptr())) throw py::type_error("to_units takes physical numbers (got dtype object)"); + const auto np = py::module_::import("numpy"); + const auto raw = py::reinterpret_steal(checked(PyObject_CallOneArg(np.attr("asarray").ptr(), x.ptr())).release()); + const char kind = raw.dtype().kind(); + if (kind != 'f' && kind != 'i' && kind != 'u') throw py::type_error(dtype_text(raw)); + if (raw.ndim() > 1) throw py::type_error(kNotOneDimensional); + throw py::type_error(kNotPhysical); +} + +// to_units' x as doubles: one scalar, or the entries of a 1-D input. +struct PhysicalValues { + bool scalar {false}; + std::vector values; +}; + +// The one classifier of to_units' x, in this order: +// 1. a bool, a str or a bytes: TypeError (the scalar text); +// 2. an int or a float, a subclass included, by its stored value; a NumPy integer or floating +// scalar (scalar_kind): one number (an int beyond 64 bits: ValueError); +// 3. a NumPy array: an integer or floating dtype (else TypeError, the dtype text), of rank 0 (one +// number) or 1 (its entries; another rank TypeError, the shape text); +// 4. a Python sequence (a memoryview of rank 1; a str or bytes never reaches here), each entry +// through scalar_kind: any entry of another type refuses it as NumPy words it; then NumPy's +// dtype: bools alone TypeError with `got dtype bool`, an int beyond 64 bits `got dtype +// object` (ValueError among numbers alone, TypeError beside a bool); then bools among +// numbers TypeError (the scalar text: NumPy would read them as numbers); numbers alone are +// its values; +// 5. anything else: TypeError, as NumPy words it (refuse_physical). +// An int beyond 64 bits among numbers is a value outside the domain (ValueError, NumPy's `got dtype +// object`), decided after every entry is classified. No Python object is made but the checked +// copies of NumPy integers, NumPy's array of a given array and the refusal's reading. +PhysicalValues physical_values(const py::handle& x) { + PhysicalValues out; + if (py::isinstance(x) || py::isinstance(x)) throw py::type_error(kNotPhysical); + constexpr const char* wide = "to_units takes physical numbers (got dtype object)"; + double value = 0.0; + switch (scalar_kind(x, value)) { + case Kind::Bool: + throw py::type_error(kNotPhysical); + case Kind::Wide: + throw py::value_error(wide); + case Kind::Number: + out.scalar = true; + out.values.push_back(value); + return out; + case Kind::Other: + break; + } + if (py::isinstance(x)) { + const auto given = py::reinterpret_borrow(x); + const char kind = given.dtype().kind(); + if (kind != 'f' && kind != 'i' && kind != 'u') throw py::type_error(dtype_text(given)); + if (given.ndim() > 1) throw py::type_error(kNotOneDimensional); + const auto doubles = numpy_conversion(given); + out.scalar = given.ndim() == 0; + out.values.assign(doubles.data(), doubles.data() + doubles.size()); + return out; + } + if (PySequence_Check(x.ptr()) == 0) refuse_physical(x); + if (PyMemoryView_Check(x.ptr()) != 0 && PyMemoryView_GET_BUFFER(x.ptr())->ndim != 1) { + if (PyMemoryView_GET_BUFFER(x.ptr())->ndim > 1) throw py::type_error(kNotOneDimensional); + refuse_physical(x); + } + // A list or a tuple is read in place; any other sequence by index, as NumPy reads it (its + // own __len__ and __getitem__; no iterator, whose machinery may absorb a failure). + const bool in_place = PyList_Check(x.ptr()) != 0 || PyTuple_Check(x.ptr()) != 0; + const Py_ssize_t size = PySequence_Size(x.ptr()); + if (size < 0) throw py::error_already_set(); + const auto n = static_cast(size); + const auto entry = [&](std::size_t i) { + if (in_place) return py::reinterpret_borrow(PySequence_Fast_ITEMS(x.ptr())[i]); + return checked(PySequence_GetItem(x.ptr(), static_cast(i))); + }; + // A refusal is worded on the entries as a list: the sequence itself, or a checked copy. + const auto refuse = [&]() { + if (in_place) refuse_physical(x); + const py::object copy = checked(PyList_New(size)); + for (std::size_t i = 0; i < n; ++i) PyList_SET_ITEM(copy.ptr(), static_cast(i), entry(i).release().ptr()); + refuse_physical(copy); + }; + out.values.reserve(n); + std::size_t bools = 0; + std::size_t wides = 0; + for (std::size_t i = 0; i < n; ++i) { + switch (scalar_kind(entry(i), value)) { + case Kind::Number: + out.values.push_back(value); + break; + case Kind::Wide: + ++wides; + break; + case Kind::Bool: + ++bools; + break; + case Kind::Other: + refuse(); + } + } + // NumPy's order: its dtype first (bools alone: bool; an int beyond 64 bits: object, a value + // error only among numbers alone), then a bool among numbers. + if (bools == n && n > 0) throw py::type_error("to_units takes physical numbers (got dtype bool)"); + if (wides > 0) { + if (bools > 0) throw py::type_error(wide); + throw py::value_error(wide); + } + if (bools > 0) throw py::type_error(kNotPhysical); + return out; +} + +// x * 2^k for an unbounded Python int, correctly rounded: Python's int shift and int true +// division are exact or correctly rounded, so only a result beyond the finite doubles fails. +double scaled_python_int(const py::object& x, int k) { + try { + py::object r; + if (k >= 0) { + r = py::float_(x.attr("__lshift__")(py::int_(k))); + } else { + r = x.attr("__truediv__")(py::int_(1).attr("__lshift__")(py::int_(-k))); + } + return r.cast(); + } catch (py::error_already_set& e) { + if (e.matches(PyExc_OverflowError)) { + throw py::value_error("as_float: " + std::string(py::str(x)) + + " units times the resolution exceed the largest finite double"); + } + throw; + } +} + +// One Amount to a physical double; the resolution's exponent k is already validated. +double amount_as_float(const py::handle& item, int k, double resolution) { + py::object idx = python_int(item, "as_float: x"); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow == 0) return as_float(static_cast(v), resolution); + return scaled_python_int(idx, k); +} + +} // namespace + +void register_units(py::module_& m) { + m.def( + "to_units", + [](const py::object& x, StrictFloat resolution) -> py::object { + const int k = resolution_exponent(resolution); // before the shape: an empty array is no exemption + const PhysicalValues v = physical_values(x); + if (v.scalar) return int_object(to_units(v.values[0], resolution)); + // At this scale the nearest double below 2^62 is hundreds of units + // away: rounding cannot cross the boundary. Validate with the physical + // bound, then leave the scaling/rounding kernel to the detached phase. + const double limit = 4611686018427387904.0 * static_cast(resolution); + for (double x : v.values) { + if (!(x >= 0.0 && x < limit)) (void)to_units(x, resolution); + } + auto result = native_call([&] { + std::vector values(v.values.size()); + for (std::size_t i = 0; i < v.values.size(); ++i) { + values[i] = static_cast(detail::rounded_units(v.values[i], k)); + } + return values; + }); + return int64_array(result); + }, + py::arg("x"), py::arg("resolution"), + "Physical value(s) to whole units of the resolution q: each input is read as its binary64 value b,\n" + "and b/q is rounded half up to an integer, exactly. Converted back, a result differs from b by at\n" + "most q/2 (+q/2 exactly at a tie, which goes up).\n" + "float32 widens exactly; integers become the nearest double, ties to even, so an integer above 2^53\n" + "can change before the rounding (2^53+1 at q=1 gives 2^53). The result is floor(b/q + 1/2),\n" + "evaluated exactly, never by floating addition of 1/2 or by reading the printed decimal. Ties are\n" + "b=(2n+1)*q/2, 0 <= n < 2^52; each gives n+1, never the even neighbour.\n" + "A scalar or 0-d array gives an int; a sequence or 1-d array gives an int64 array. Negative\n" + "zero gives 0. NaN, infinity, negative values, an integer too large for NumPy and results\n" + "at or above 2^62 units raise ValueError. A bool, string, wrong dtype/rank or type other\n" + "than int, float or NumPy numbers raises TypeError; int/float subclasses use stored values.\n" + "q must be a power of two in [2^-32, 2^32]: a float, NumPy float or integer (another type\n" + "raises TypeError, an integer beyond the float range ValueError).\n" + "N separately rounded values have a total error in (-N*q/2, N*q/2], with no promised sign: round a\n" + "demand once, then split whole units. A decimal and the double it parses to give the same units\n" + "only when both lie in the same half-up cell: 5501195.1796 at q=2^-21 gives 11536842473289, whereas\n" + "its decimal reading would give 11536842473288 (docs/semantics.md, section 1, has the full rule).\n" + "CostStretch deliberately reads printed decimals for ratios; physical amounts use b."); + + m.def( + "as_float", + [](const py::object& x, StrictFloat resolution) -> py::object { + const int k = resolution_exponent(resolution); // the resolution before the shape + if (!is_bool(x) && + (PyLong_Check(x.ptr()) != 0 || has_type(x.ptr(), numpy_number_types().first))) { + return py::float_(amount_as_float(x, k, resolution)); + } + // A sequence, a NumPy array of any rank included: wide_integers refuses a rank other than 1. + if (!is_invalid_sequence(x)) { + const auto v = wide_integers(x, "as_float: x"); + std::vector values(v.size()); + // Arbitrary Python integer arithmetic remains attached; the ordinary + // fixed-width conversion loop works only on the copied native values. + for (std::size_t i = 0; i < v.size(); ++i) { + if (v.sign(i) != 0) values[i] = amount_as_float(v.source[py::int_(i)], k, resolution); + } + native_call([&] { + for (std::size_t i = 0; i < v.size(); ++i) { + if (v.sign(i) == 0) values[i] = detail::scaled_units(v.value[i], k); + } + }); + py::array_t out(static_cast(values.size())); + std::copy(values.begin(), values.end(), out.mutable_data()); + return std::move(out); + } + return py::float_(amount_as_float(x, k, resolution)); + }, + py::arg("x"), py::arg("resolution"), + "Units to physical value(s): the exact product x*q correctly rounded to binary64, nearest\n" + "with ties to even; exact for abs(x) <= 2^53. x is any signed, unbounded integer: an int or NumPy integer gives a\n" + "float, an integer vector a float64 array. An int or NumPy integer, a subclass included,\n" + "is never read as a sequence. An integer position accepts __index__, never\n" + "a bool; other types raise TypeError. q is a power of two in [2^-32, 2^32] (float, NumPy\n" + "float or integer). The result is finite iff abs(x*q) < 2^1024 - 2^970; otherwise raise\n" + "ValueError('as_float: units times the resolution exceed the largest finite double').\n" + "Exact for abs(x) <= 2^53 inclusive. Above 2^53 an integer of binary length L is exact\n" + "iff divisible by 2^(L-53); 2^53+1 rounds to 2^53, 2^53+3 to 2^53+4, times q.\n" + "to_units(as_float(x,q),q)==x holds for x in [0,2^53] at every q, and for larger x in\n" + "[0,2^62) exactly when x is a double. The first failure is 2^53+1; 2^62-1 rounds to\n" + "2^62*q and reconversion raises the to_units domain ValueError. A successful to_units\n" + "result is always exact here; its error against the promoted binary64 input is in\n" + "(-q/2,q/2]. Choose q before Graph construction, then use Graph.resolution()."); +} + +} // namespace netgraph::core::bindings diff --git a/bindings/python/values.hpp b/bindings/python/values.hpp new file mode 100644 index 0000000..d3de1ab --- /dev/null +++ b/bindings/python/values.hpp @@ -0,0 +1,318 @@ +/* + Small conversions shared by the registration files: read-only NumPy views that + keep their owner alive, the resolution's check, exact counters, int64 arrays, FlowIds, + (EdgeId, units) pairs, amounts (checked at once or kept as given) and the type checks of option + arguments. +*/ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "netgraph/core/eligibility.hpp" +#include "netgraph/core/flow_record.hpp" +#include "netgraph/core/flow_policy.hpp" +#include "netgraph/core/graph.hpp" +#include "netgraph/core/types.hpp" +#include "integers.hpp" + +namespace py = pybind11; + +namespace netgraph::core::bindings { + +// Read-only NumPy view of a span owned by `owner`. +template +py::array readonly_view(std::span s, const py::handle& owner, const py::dtype& dtype) { + static constexpr T empty_element {}; + py::array arr(dtype, {static_cast(s.size())}, {static_cast(sizeof(T))}, + s.empty() ? &empty_element : s.data(), owner); + py::detail::array_proxy(arr.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + return arr; +} + +template +py::array readonly_view(std::span s, const py::handle& owner) { + return readonly_view(s, owner, py::dtype::of()); +} + +// Validates a resolution (a power of two in [2^-32, 2^32]) and returns its exponent. +inline int resolution_exponent(double resolution) { + (void)as_float(0, resolution); // throws the resolution message + return std::ilogb(resolution); +} + +// A new reference from the C API, or the pending Python error raised (pybind11's own int and +// tuple constructors fail on a failed allocation with an unchecked internal error instead). +inline py::object checked(PyObject* o) { + if (o == nullptr) throw py::error_already_set(); + return py::reinterpret_steal(o); +} + +// hi * 2^64 + lo as an exact Python int. +inline py::object counter_int(const Counter128& c) { + py::object lo = checked(PyLong_FromUnsignedLongLong(c.lo)); + if (c.hi == 0) return lo; + py::object hi = checked(PyLong_FromUnsignedLongLong(c.hi)); + py::object shift = checked(PyLong_FromLong(64)); + py::object high = checked(PyNumber_Lshift(hi.ptr(), shift.ptr())); + return checked(PyNumber_Or(high.ptr(), lo.ptr())); +} + +// The readers of fields and properties build every Python value below through the checked C API +// (a new reference, or the pending MemoryError raised). pybind11's own conversions report a failed +// allocation as another class: its return conversion of a C++ value as TypeError, its int, float, +// list and tuple constructors as RuntimeError. + +// An integer (any integer type but bool) or a 128-bit counter as a Python int. +template +py::int_ int_object(T v) { + static_assert(std::is_integral_v && !std::is_same_v); + PyObject* o = nullptr; + if constexpr (std::is_signed_v) { + o = PyLong_FromLongLong(static_cast(v)); + } else { + o = PyLong_FromUnsignedLongLong(static_cast(v)); + } + return py::reinterpret_steal(checked(o).release()); +} +inline py::int_ int_object(const Counter128& c) { return py::reinterpret_steal(counter_int(c).release()); } + +inline py::float_ float_object(double v) { return py::reinterpret_steal(checked(PyFloat_FromDouble(v)).release()); } + +// None, or the value as a Python int. +template +py::typing::Optional optional_int(const std::optional& v) { + return py::reinterpret_steal>(v.has_value() ? int_object(*v).release() : py::none().release()); +} + +// The reader of an integer field, and of an optional one (None when absent). +template +auto int_reader(T C::*field) { + return [field](const C& c) { return int_object(c.*field); }; +} +template +auto optional_int_reader(std::optional C::*field) { + return [field](const C& c) { return optional_int(c.*field); }; +} + +// (a, b) as a tuple of two Python ints. +template +py::typing::Tuple int_pair(const A& a, const B& b) { + py::object x = int_object(a); + py::object y = int_object(b); + PyObject* out = checked(PyTuple_New(2)).release().ptr(); + PyTuple_SET_ITEM(out, 0, x.release().ptr()); + PyTuple_SET_ITEM(out, 1, y.release().ptr()); + return py::reinterpret_steal>(out); +} + +// Integers as a list of Python ints. +template +py::typing::List ints_list(std::span v) { + auto out = py::reinterpret_steal>(checked(PyList_New(static_cast(v.size()))).release()); + for (std::size_t i = 0; i < v.size(); ++i) PyList_SET_ITEM(out.ptr(), static_cast(i), int_object(v[i]).release().ptr()); + return out; +} + +// (a, b) pairs as a list (Out: its pybind11 type, which names the stub's type) or a tuple of +// two-int tuples. +template +Out pairs_list(const std::vector>& v) { + auto out = py::reinterpret_steal(checked(PyList_New(static_cast(v.size()))).release()); + for (std::size_t i = 0; i < v.size(); ++i) { + PyList_SET_ITEM(out.ptr(), static_cast(i), int_pair(v[i].first, v[i].second).release().ptr()); + } + return out; +} +template +py::tuple pairs_tuple(const std::vector>& v) { + auto out = py::reinterpret_steal(checked(PyTuple_New(static_cast(v.size()))).release()); + for (std::size_t i = 0; i < v.size(); ++i) { + PyTuple_SET_ITEM(out.ptr(), static_cast(i), int_pair(v[i].first, v[i].second).release().ptr()); + } + return out; +} + +// A tuple of the given objects (made by the caller), through a checked allocation. +inline py::tuple objects_tuple(std::initializer_list items) { + auto out = py::reinterpret_steal(checked(PyTuple_New(static_cast(items.size()))).release()); + Py_ssize_t i = 0; + for (const auto& item : items) PyTuple_SET_ITEM(out.ptr(), i++, item.inc_ref().ptr()); + return out; +} + +// fraction_type(n, d) (fractions.Fraction): the two ints through checked allocations and the call +// through the C API, so a failure of the call raises as raised. +inline py::object fraction_object(const py::handle& fraction_type, std::int64_t n, std::int64_t d) { + const py::object num = int_object(n); + const py::object den = int_object(d); + return checked(PyObject_CallFunctionObjArgs(fraction_type.ptr(), num.ptr(), den.ptr(), nullptr)); +} + +// A str through a checked allocation. +inline py::str str_object(const char* text) { return py::reinterpret_steal(checked(PyUnicode_FromString(text)).release()); } + +// The hash of `key` as a Python int: a failure of the hash raises as raised, and the int is made +// through a checked allocation (pybind11's return conversion of a hash reports a failed +// allocation as TypeError). +inline py::int_ hash_int(const py::handle& key) { + const Py_hash_t h = PyObject_Hash(key.ptr()); + if (h == -1 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + return int_object(static_cast(h)); +} + +// Integers as a tuple of Python ints. +inline py::tuple ints_tuple(std::span v) { + auto out = py::reinterpret_steal(checked(PyTuple_New(static_cast(v.size()))).release()); + for (std::size_t i = 0; i < v.size(); ++i) { + PyTuple_SET_ITEM(out.ptr(), static_cast(i), checked(PyLong_FromLong(v[i])).release().ptr()); + } + return out; +} + +// A tuple of Python ints, each through a checked allocation. +inline py::tuple amounts_tuple(std::span v) { + auto out = py::reinterpret_steal(checked(PyTuple_New(static_cast(v.size()))).release()); + for (std::size_t i = 0; i < v.size(); ++i) { + PyTuple_SET_ITEM(out.ptr(), static_cast(i), checked(PyLong_FromLongLong(v[i])).release().ptr()); + } + return out; +} + +inline py::array_t int64_array(const std::vector& v) { + py::array_t out(static_cast(v.size())); + std::copy(v.begin(), v.end(), out.mutable_data()); + return out; +} + +inline FlowId flow_id_of(const py::object& obj) { + // Not a tuple is a wrong type, a tuple of another length a wrong value: one text. + if (!py::isinstance(obj)) throw py::type_error("a FlowId is a tuple (ledger uid, id)"); + if (py::len(obj) != 2) throw py::value_error("a FlowId is a tuple (ledger uid, id)"); + auto t = py::reinterpret_borrow(obj); + py::object uid = python_int(t[0], "FlowId ledger uid"); + py::object id = python_int(t[1], "FlowId id"); + int o1 = 0, o2 = 0; + const unsigned long long u = PyLong_AsUnsignedLongLong(uid.ptr()); + if (PyErr_Occurred()) { + // Only the range error means no such ledger; another failure (memory running out while that + // error is raised) propagates as raised. + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) throw py::error_already_set(); + PyErr_Clear(); + o1 = 1; + } + const long long i = PyLong_AsLongLongAndOverflow(id.ptr(), &o2); + if (o1 != 0 || o2 != 0 || i < 0) { + throw py::value_error("FlowId (" + std::string(py::str(uid)) + ", " + std::string(py::str(id)) + ") is retired or unknown"); + } + return FlowId {static_cast(u), static_cast(i)}; +} + + +// An amount argument: an integer number of units in [0, 2^62) or +inf (float('inf')); nullopt is +// +inf. Every other argument raises the one `text`, with the class of its cause: a type the position +// never takes (a bool, a string, None, another non-number) TypeError; a float other than +inf (NaN, +// -inf, a finite one) or an integer out of range ValueError. An exception of the object's own +// __index__ propagates as raised. +inline std::optional units_or_inf(const py::object& value, const char* text) { + if (PyFloat_Check(value.ptr()) || has_type(value.ptr(), numpy_number_types().second)) { + const double x = PyFloat_AsDouble(value.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + if (std::isinf(x) && x > 0) return std::nullopt; + throw py::value_error(text); + } + if (is_bool(value) || !PyIndex_Check(value.ptr())) throw py::type_error(text); + const py::object idx = py::reinterpret_steal(PyNumber_Index(value.ptr())); + if (!idx) throw py::error_already_set(); + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(idx.ptr(), &overflow); + if (overflow != 0 || v < 0 || v >= kDomainLimit) throw py::value_error(text); + return static_cast(v); +} + +// An amount kept as given (Units: an int or a float; None: absent): an action's offer and design +// demand, an EvaluationOptions option. A value of another type (a bool, a string, ...) raises +// TypeError here; its domain is the call's check. An __index__ object is kept as its int. +inline py::object units_as_given(const py::object& obj, const char* name) { + if (obj.is_none() || PyFloat_Check(obj.ptr()) || has_type(obj.ptr(), numpy_number_types().second)) { + return obj; + } + if (!is_bool(obj) && PyIndex_Check(obj.ptr())) { + auto idx = py::reinterpret_steal(PyNumber_Index(obj.ptr())); + if (!idx) throw py::error_already_set(); + return idx; + } + throw py::type_error(std::string(name) + " must be an int or a float (got " + + std::string(py::str(py::type::of(obj).attr("__name__"))) + ")"); +} + +// The C++ form of an amount kept as given: None absent; +inf the inner nullopt; an int in int64 its +// value; any other value (a float other than +inf, an int beyond int64) -1, which is outside every +// domain and so raises the amount's domain error in its place. +inline std::optional> units_at_use(const py::object& obj) { + if (obj.is_none()) return std::nullopt; + if (PyLong_Check(obj.ptr())) { + int overflow = 0; + const long long v = PyLong_AsLongLongAndOverflow(obj.ptr(), &overflow); + if (overflow != 0) return Amount {-1}; + return Amount {v}; + } + const double x = PyFloat_AsDouble(obj.ptr()); + if (x == -1.0 && PyErr_Occurred() != nullptr) throw py::error_already_set(); + if (std::isinf(x) && x > 0) return std::optional> {std::in_place, std::nullopt}; + return Amount {-1}; +} + +// A black-hole set argument: an Eligibility or None (its graph is checked by the call). +inline std::shared_ptr eligibility_or_none(const py::object& obj, const char* name) { + if (obj.is_none()) return nullptr; + if (!py::isinstance(obj)) { + throw py::type_error(std::string(name) + " must be an Eligibility or None (got " + + std::string(py::str(py::type::of(obj).attr("__name__"))) + ")"); + } + return obj.cast>(); +} + +inline OutputOptions outputs_of(const py::object& obj) { + if (obj.is_none()) return OutputOptions {}; + if (!py::isinstance(obj)) { + throw py::type_error("outputs must be an OutputOptions (got " + std::string(py::str(py::type::of(obj).attr("__name__"))) + + ")"); + } + return obj.cast(); +} + +template +Options options_of(const py::object& obj, const char* name) { + if (!py::isinstance(obj)) { + throw py::type_error(std::string(name) + " expected (got " + std::string(py::str(py::type::of(obj).attr("__name__"))) + + ")"); + } + return obj.cast(); +} + +// A cost ceiling argument (DistributedTE's and KspOptions'): None, an AbsoluteCost or a CostStretch; +// any other object raises one TypeError. +inline std::optional cost_ceiling_of(const py::object& obj) { + if (obj.is_none()) return std::nullopt; + if (py::isinstance(obj)) return CostCeiling {obj.cast()}; + if (py::isinstance(obj)) return CostCeiling {obj.cast()}; + throw py::type_error("cost_ceiling must be AbsoluteCost, CostStretch or None"); +} + +} // namespace netgraph::core::bindings diff --git a/dev/benchmark_profiling_overhead.py b/dev/benchmark_profiling_overhead.py deleted file mode 100644 index 1a32e68..0000000 --- a/dev/benchmark_profiling_overhead.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark for measuring netgraph_core performance and profiling overhead. - -Runs SPF and flow placement operations on a mesh graph topology. -Useful for: - - Measuring profiling infrastructure overhead (compare with/without NGRAPH_CORE_PROFILE=1) - - Regression testing after code changes - - Comparing performance across versions - -Usage: - # Default run (400-node mesh, 500 SPF + 100 flow ops) - python dev/benchmark_profiling_overhead.py - - # Quick smoke test - python dev/benchmark_profiling_overhead.py --quick - - # Custom parameters - python dev/benchmark_profiling_overhead.py --mesh-size 30 --spf-iters 1000 --flow-iters 200 - - # Compare profiling overhead - python dev/benchmark_profiling_overhead.py # baseline - NGRAPH_CORE_PROFILE=1 python dev/benchmark_profiling_overhead.py # with profiling - - # JSON output for scripting - python dev/benchmark_profiling_overhead.py --json -""" - -import argparse -import json -import statistics -import sys -import time - -import numpy as np - -import netgraph_core as ngc - - -def create_mesh_graph(n: int) -> ngc.StrictMultiDiGraph: - """Create an n x n mesh graph with bidirectional edges. - - Args: - n: Grid dimension (creates n*n nodes) - - Returns: - Graph with 4-connected mesh topology, unit costs, 100.0 capacity per edge. - """ - num_nodes = n * n - edges = [] - - for row in range(n): - for col in range(n): - node = row * n + col - if col < n - 1: - right = row * n + (col + 1) - edges.append((node, right, 1, 100.0)) - edges.append((right, node, 1, 100.0)) - if row < n - 1: - down = (row + 1) * n + col - edges.append((node, down, 1, 100.0)) - edges.append((down, node, 1, 100.0)) - - src = np.array([e[0] for e in edges], dtype=np.int32) - dst = np.array([e[1] for e in edges], dtype=np.int32) - cost = np.array([e[2] for e in edges], dtype=np.int64) - cap = np.array([e[3] for e in edges], dtype=np.float64) - ext_edge_ids = np.arange(len(edges), dtype=np.int64) - - return ngc.StrictMultiDiGraph.from_arrays( - num_nodes, src, dst, cap, cost, ext_edge_ids - ) - - -def benchmark_spf( - graph_handle, algs: ngc.Algorithms, num_nodes: int, iterations: int -) -> list[float]: - """Run SPF iterations and return per-call times in ms.""" - times = [] - edge_sel = ngc.EdgeSelection( - multi_edge=False, - require_capacity=False, - tie_break=ngc.EdgeTieBreak.DETERMINISTIC, - ) - - for i in range(iterations): - src = i % num_nodes - start = time.perf_counter() - algs.spf(graph_handle, src, selection=edge_sel) - times.append((time.perf_counter() - start) * 1000) - - return times - - -def benchmark_flow_placement( - g: ngc.StrictMultiDiGraph, graph_handle, algs: ngc.Algorithms, iterations: int -) -> list[float]: - """Run flow placement iterations and return per-call times in ms.""" - num_nodes = g.num_nodes() - times = [] - - for i in range(iterations): - fg = ngc.FlowGraph(g) - cfg = ngc.FlowPolicyConfig() - cfg.flow_placement = ngc.FlowPlacement.EQUAL_BALANCED - cfg.max_flow_count = 16 - cfg.min_flow_count = 16 - cfg.selection = ngc.EdgeSelection( - multi_edge=False, - require_capacity=True, - tie_break=ngc.EdgeTieBreak.PREFER_HIGHER_RESIDUAL, - ) - cfg.require_capacity = True - cfg.multipath = False - cfg.reoptimize_flows_on_each_placement = True - policy = ngc.FlowPolicy(algs, graph_handle, cfg) - - src = i % num_nodes - dst = (num_nodes - 1 - i) % num_nodes - if src == dst: - dst = (dst + 1) % num_nodes - - start = time.perf_counter() - policy.place_demand(fg, src, dst, 0, 1000.0) - times.append((time.perf_counter() - start) * 1000) - - return times - - -def compute_stats(name: str, times: list[float]) -> dict: - """Compute statistics for a benchmark run.""" - return { - "name": name, - "iterations": len(times), - "mean_ms": statistics.mean(times), - "median_ms": statistics.median(times), - "stdev_ms": statistics.stdev(times) if len(times) > 1 else 0, - "min_ms": min(times), - "max_ms": max(times), - "total_ms": sum(times), - } - - -def print_stats(stats: dict) -> None: - """Print benchmark statistics.""" - print(f"\n{stats['name']}:") - print(f" Iterations: {stats['iterations']}") - print(f" Mean: {stats['mean_ms']:.3f} ms") - print(f" Median: {stats['median_ms']:.3f} ms") - print(f" Stdev: {stats['stdev_ms']:.3f} ms") - print(f" Min: {stats['min_ms']:.3f} ms") - print(f" Max: {stats['max_ms']:.3f} ms") - print(f" Total: {stats['total_ms']:.1f} ms") - - -def main(): - parser = argparse.ArgumentParser( - description="Benchmark netgraph_core performance and profiling overhead.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - %(prog)s # default benchmark - %(prog)s --quick # fast smoke test - %(prog)s --mesh-size 30 # larger graph - %(prog)s --json # machine-readable output - NGRAPH_CORE_PROFILE=1 %(prog)s # measure with profiling enabled -""", - ) - parser.add_argument( - "--mesh-size", - type=int, - default=20, - metavar="N", - help="mesh grid size (N x N nodes, default: 20)", - ) - parser.add_argument( - "--spf-iters", - type=int, - default=500, - metavar="N", - help="number of SPF iterations (default: 500)", - ) - parser.add_argument( - "--flow-iters", - type=int, - default=100, - metavar="N", - help="number of flow placement iterations (default: 100)", - ) - parser.add_argument( - "--quick", - action="store_true", - help="quick run with minimal iterations (mesh=10, spf=50, flow=10)", - ) - parser.add_argument( - "--json", - action="store_true", - help="output results as JSON (for scripting)", - ) - args = parser.parse_args() - - if args.quick: - args.mesh_size = 10 - args.spf_iters = 50 - args.flow_iters = 10 - - profiling_enabled = getattr(ngc, "profiling_enabled", lambda: None)() - - if not args.json: - print("=" * 60) - print("NetGraph-Core Benchmark") - print("=" * 60) - print(f"\nVersion: {ngc.__version__}") - if profiling_enabled is not None: - print(f"Profiling: {'ON' if profiling_enabled else 'OFF'}") - print( - f"\nGraph: {args.mesh_size}x{args.mesh_size} mesh ({args.mesh_size**2} nodes)" - ) - print(f"SPF iterations: {args.spf_iters}") - print(f"Flow iterations: {args.flow_iters}") - print("\nCreating graph...", end=" ", flush=True) - - g = create_mesh_graph(args.mesh_size) - num_nodes = g.num_nodes() - num_edges = g.num_edges() - - if not args.json: - print(f"done ({num_nodes} nodes, {num_edges} edges)") - - backend = ngc.Backend.cpu() - algs = ngc.Algorithms(backend) - graph_handle = algs.build_graph(g) - - # Warmup (results discarded, profiling stats reset after) - if not args.json: - print("\nWarmup...", end=" ", flush=True) - sys.stdout.flush() - _ = benchmark_spf(graph_handle, algs, num_nodes, min(10, args.spf_iters)) - _ = benchmark_flow_placement(g, graph_handle, algs, min(5, args.flow_iters)) - if hasattr(ngc, "profiling_reset"): - ngc.profiling_reset() - if not args.json: - print("done", flush=True) - print("\n" + "-" * 40) - print("Running benchmarks...", flush=True) - sys.stdout.flush() - - spf_times = benchmark_spf(graph_handle, algs, num_nodes, args.spf_iters) - spf_stats = compute_stats("Shortest Paths (SPF)", spf_times) - - flow_times = benchmark_flow_placement(g, graph_handle, algs, args.flow_iters) - flow_stats = compute_stats("Flow Placement", flow_times) - - total_ops = args.spf_iters + args.flow_iters - total_time_ms = spf_stats["total_ms"] + flow_stats["total_ms"] - throughput = total_ops / (total_time_ms / 1000) if total_time_ms > 0 else 0 - - if args.json: - result = { - "version": ngc.__version__, - "profiling_enabled": profiling_enabled, - "config": { - "mesh_size": args.mesh_size, - "num_nodes": num_nodes, - "num_edges": num_edges, - "spf_iterations": args.spf_iters, - "flow_iterations": args.flow_iters, - }, - "spf": {k: v for k, v in spf_stats.items() if k != "name"}, - "flow": {k: v for k, v in flow_stats.items() if k != "name"}, - "summary": { - "total_ops": total_ops, - "total_time_ms": total_time_ms, - "throughput_ops_per_sec": throughput, - }, - } - print(json.dumps(result, indent=2)) - else: - print_stats(spf_stats) - print_stats(flow_stats) - - print("\n" + "=" * 60) - print("SUMMARY") - print("=" * 60) - print(f"Total operations: {total_ops}") - print(f"Total time: {total_time_ms:.1f} ms ({total_time_ms / 1000:.2f} s)") - print(f"Throughput: {throughput:.1f} ops/sec") - - if profiling_enabled: - print("\n" + "-" * 40) - print("Profiling stats:", flush=True) - sys.stdout.flush() - ngc.profiling_dump() - sys.stderr.flush() - - -if __name__ == "__main__": - main() diff --git a/dev/coverage_summary.py b/dev/coverage_summary.py index ecd1b33..ae18ad9 100644 --- a/dev/coverage_summary.py +++ b/dev/coverage_summary.py @@ -10,46 +10,93 @@ Defaults: PY_XML = build/coverage/coverage-python.xml CPP_XML = build/coverage/coverage-cpp.xml + +A missing, unreadable or malformed report is an error (exit 1), never zero coverage, and so is a +report that counts no line, which would otherwise show as 100%: gcovr writes one, and exits 0, +when it reads no profile data (the profiles are not where it looks, or the filters match no +file), and coverage.py when the files it measured hold no statement. A report carries both +branch totals or neither, and covers at most what it counts. Each total is a count written in +decimal digits, optionally with a zero fraction ("12" or "12.0"), and at most 2**63 - 1 (far +beyond any real report); anything else (a sign, a nonzero fraction, an exponent, NaN or +infinity) is malformed. The counts are read and compared exactly. """ from __future__ import annotations +import argparse import html import os +import re import sys import xml.etree.ElementTree as ET from typing import Optional, Tuple +# A count: decimal digits, optionally with a zero fraction (some tools write "12.0"). +_COUNT = re.compile(r"([0-9]+)(?:\.0*)?") +_MAX_COUNT = 2**63 - 1 + + +class ReportError(Exception): + """A coverage report is missing, unreadable or malformed.""" + def parse_cobertura_totals( xml_path: str, ) -> Tuple[int, int, Optional[int], Optional[int]]: """Return (lines_valid, lines_covered, branches_valid, branches_covered). - Missing values are returned as None. If the file is missing, returns zeros/None. + The line totals are required; the branch totals are both None when the report has none. + Raises ReportError when the file is missing or unreadable, is not a Cobertura report, + holds a total that is not a count (the module docstring's spelling and bound), has one + branch total without the other, covers more lines or branches than it counts, or counts + no line (it measured nothing). """ - if not os.path.exists(xml_path): - return 0, 0, None, None - try: root = ET.parse(xml_path).getroot() - except Exception: - return 0, 0, None, None + except (OSError, ET.ParseError) as exc: + raise ReportError(f"{xml_path}: cannot read the report: {exc}") from exc + if root.tag != "coverage": + raise ReportError(f"{xml_path}: not a Cobertura report (root <{root.tag}>)") - def _to_int(value: Optional[str]) -> Optional[int]: + def _to_int(name: str, required: bool) -> Optional[int]: + value = root.attrib.get(name) if value is None: + if required: + raise ReportError(f"{xml_path}: no {name} total") return None - try: - # Some tools may write floats; coerce safely - return int(float(value)) - except Exception: - return None - - lines_valid = _to_int(root.attrib.get("lines-valid")) or 0 - lines_covered = _to_int(root.attrib.get("lines-covered")) or 0 - branches_valid = _to_int(root.attrib.get("branches-valid")) - branches_covered = _to_int(root.attrib.get("branches-covered")) - + match = _COUNT.fullmatch(value) + if match is None: + raise ReportError( + f"{xml_path}: {name}={value!r} is not a count (a nonnegative integer)" + ) + digits = match.group(1).lstrip("0") or "0" + if len(digits) > len(str(_MAX_COUNT)) or int(digits) > _MAX_COUNT: + raise ReportError( + f"{xml_path}: {name}={value!r} is above the largest count, 2**63 - 1" + ) + return int(digits) + + lines_valid = _to_int("lines-valid", True) + lines_covered = _to_int("lines-covered", True) + branches_valid = _to_int("branches-valid", False) + branches_covered = _to_int("branches-covered", False) + assert lines_valid is not None and lines_covered is not None + if lines_covered > lines_valid: + raise ReportError(f"{xml_path}: {lines_covered} lines covered of {lines_valid}") + if lines_valid == 0: + raise ReportError( + f"{xml_path}: the report counts no line (no coverage data was read, or the" + " measured files hold no statement)" + ) + if branches_valid is None or branches_covered is None: + if branches_valid is not None or branches_covered is not None: + raise ReportError( + f"{xml_path}: branches-valid and branches-covered must both be present or both absent" + ) + elif branches_covered > branches_valid: + raise ReportError( + f"{xml_path}: {branches_covered} branches covered of {branches_valid}" + ) return lines_valid, lines_covered, branches_valid, branches_covered @@ -70,10 +117,8 @@ def format_text_table( total_lines_valid = py_lines_valid + cpp_lines_valid total_lines_cov = py_lines_cov + cpp_lines_cov - # Branch totals only if present in both - have_branches = (py_br_valid is not None and py_br_cov is not None) or ( - cpp_br_valid is not None and cpp_br_cov is not None - ) + # Branch columns when either report has branch totals; a report without them counts 0 of 0. + have_branches = py_br_valid is not None or cpp_br_valid is not None total_br_valid = (py_br_valid or 0) + (cpp_br_valid or 0) if have_branches else 0 total_br_cov = (py_br_cov or 0) + (cpp_br_cov or 0) if have_branches else 0 @@ -108,31 +153,40 @@ def format_html_table(text_table: str, title: str = "Combined Coverage") -> str:

{html.escape(title)}

{pre}
""" -def main() -> int: - py_xml = ( - sys.argv[1] - if len(sys.argv) > 1 and not sys.argv[1].startswith("--") - else "build/coverage/coverage-python.xml" +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Combine the Python and C++ Cobertura reports into one summary." ) - cpp_xml = ( - sys.argv[2] - if len(sys.argv) > 2 and not sys.argv[2].startswith("--") - else "build/coverage/coverage-cpp.xml" + parser.add_argument( + "py_xml", nargs="?", default="build/coverage/coverage-python.xml" ) + parser.add_argument("cpp_xml", nargs="?", default="build/coverage/coverage-cpp.xml") + parser.add_argument( + "--html", metavar="OUT_HTML", help="also write the table as HTML" + ) + args = parser.parse_args(argv) - html_out: Optional[str] = None - for arg in sys.argv[1:]: - if arg.startswith("--html="): - html_out = arg.split("=", 1)[1] - - py_totals = parse_cobertura_totals(py_xml) - cpp_totals = parse_cobertura_totals(cpp_xml) + try: + py_totals = parse_cobertura_totals(args.py_xml) + cpp_totals = parse_cobertura_totals(args.cpp_xml) + except ReportError as exc: + print(f"coverage_summary.py: {exc}", file=sys.stderr) + return 1 text = format_text_table(py_totals, cpp_totals) print(text) + html_out: Optional[str] = args.html if html_out: - os.makedirs(os.path.dirname(html_out), exist_ok=True) - with open(html_out, "w", encoding="utf-8") as f: - f.write(format_html_table(text)) + out_dir = os.path.dirname(html_out) + try: + if out_dir: + os.makedirs(out_dir, exist_ok=True) + with open(html_out, "w", encoding="utf-8") as f: + f.write(format_html_table(text)) + except OSError as exc: + print( + f"coverage_summary.py: cannot write {html_out}: {exc}", file=sys.stderr + ) + return 1 return 0 diff --git a/dev/cpp-tests.sh b/dev/cpp-tests.sh index aab39be..b1d452a 100644 --- a/dev/cpp-tests.sh +++ b/dev/cpp-tests.sh @@ -1,27 +1,128 @@ #!/usr/bin/env bash -# Propagate configuration, build, and test failures; bound local CPU use. +# The C++ test runner (make cpp-test, cpp-test-off, sanitize-test, tsan-test, check-ci, cov). +# +# bash dev/cpp-tests.sh MODE +# +# MODE private hooks build type build directory +# release ON Release (LTO) build/cpp-tests +# off OFF Release (LTO) build/cpp-tests-off the shipped configuration +# sanitize ON Debug, ASan + UBSan build/cpp-sanitize +# tsan ON Debug, TSan build/cpp-tsan +# coverage ON Debug, --coverage build/cpp-tests-cov used by make cov +# +# Each mode has its own build directory and states every project option on the command line, so a +# value cached by an earlier configure never decides this one. One generator (Ninja), parallelism +# bounded by CMAKE_BUILD_PARALLEL_LEVEL (default 4), ctest fails when it finds no test, and every +# configuration, build or test failure fails the script. The skipped tests are checked: none in a +# hooks-on mode, exactly EXPECTED_OFF_SKIPS (the tests that need a hook) in mode off, so a test that +# silently became hooks-only is caught. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# The C++ tests that need the private hooks and skip without them (GTEST_SKIP). Inside tests that +# still run, three checks are weaker without them: the helper check_deletion +# (min_cost_flow_deletion_tests.cpp) checks each re-solve's final result, but observes each repair step +# only through the probe hook (see that file's header); PolicyFixtures.EveryFixtureReplaysIdentically +# compares no trace, and PolicyFixtures.ACorruptedExpectationFails corrupts only what that replay +# compares (see flow_policy_fixture_tests.cpp's header). +EXPECTED_OFF_SKIPS=" +ApplyActions.ForwardingSizesCountEveryParticipant +Concurrency.CacheLifecycleCannotBeOvertaken +Concurrency.FailedAdmissionLeavesNoQueuedNodeOrLease +Concurrency.LedgerReadsWaitAtTheSameGate +Concurrency.PolicyReadinessFollowsEngineInstallation +Concurrency.GateCoversExchangeAndEveryEpoch +Concurrency.FailureBeforePublicationReleasesTheGate +Concurrency.LowestWorkerFailureWins +Concurrency.PartialThreadLaunchJoinsAndReleasesLease +Concurrency.TunnelRosterChecksRunInParallelBeforeAdmission +Concurrency.TunnelPreflightReportsTheLowestErrorBeforeLaterChecks +Concurrency.TunnelPreflightJoinsAfterPartialLaunchFailure +Concurrency.NativeArgumentsAreCopiedBeforeWaiting +Concurrency.NativeEvaluatorInputsAreOwnedWhileQueued +Concurrency.NativeCacheRootsAreOwnedWhileQueued +Concurrency.FrozenTallyPublishesInsideTheLease +DeadOccurrences.ForwardingSizesWithoutThem +FlowCoreDeletion.CheckpointsEqualTheIterativeUpdate +FlowCoreDeletion.RerunningTheFailedSearchAddsExactlyOneSearch +FlowCoreDeletion.ScalarRepairStopsBeforeCancellation +ForwardingSize.TheCallChecksEachCountInOrder +GraphLayout.CombinedKeyPassesOnlyBelowNTimesKEqualsE +GraphLayout.HashAndFallbackBoundaryAt65536DistinctCosts +GraphLayout.ProbeBudgetFallbackInCollectionAndGrowth +GraphLayout.ProbeBudgetIsCumulativeAcrossTheLargeTableGrowths +GraphLayout.RadixFallbackDigitsOnBothSidesOfTheCombinedKeyBoundary +GraphLayout.RunsOfEqualCostsSkipTheLookup +Ksp.DeviationSkipsOnlySearchesComparedWithFullYen +Search.QueueDispatchAtTheWidthLimit +Sensitivity.ShortcutControlsPreserveEveryFieldAndClassifyCuts +" + +die() { echo "dev/cpp-tests.sh: $*" >&2; exit 1; } + jobs=${CMAKE_BUILD_PARALLEL_LEVEL:-4} case "$jobs" in ''|*[!0-9]*|0) echo 'CMAKE_BUILD_PARALLEL_LEVEL must be positive.' >&2; exit 2 ;; esac -case "${1:-release}" in - release) build_dir=build/cpp-tests; build_type=Release; sanitize=OFF ;; +mode=${1:-} +hooks=ON; sanitize=OFF; tsan=OFF; coverage=OFF +case "$mode" in + release) build_dir=build/cpp-tests; build_type=Release ;; + off) build_dir=build/cpp-tests-off; build_type=Release; hooks=OFF ;; sanitize) build_dir=build/cpp-sanitize; build_type=Debug; sanitize=ON ;; - *) echo 'Expected release or sanitize.' >&2; exit 2 ;; + # ThreadSanitizer (data races of the worker threads and the shared ShortestPathCache); the longest + # instrumented test takes about 30 s, inside the common timeout. + tsan) build_dir=build/cpp-tsan; build_type=Debug; tsan=ON ;; + coverage) build_dir=build/cpp-tests-cov; build_type=Debug; coverage=ON ;; + *) echo 'Usage: bash dev/cpp-tests.sh release|off|sanitize|tsan|coverage' >&2; exit 2 ;; esac + +command -v cmake >/dev/null 2>&1 || die "cmake not found on PATH (make dev installs it into venv/bin)" +command -v ctest >/dev/null 2>&1 || die "ctest not found on PATH (make dev installs it into venv/bin)" +command -v ninja >/dev/null 2>&1 || die "ninja not found on PATH (make dev installs it into venv/bin)" +cache="$build_dir/CMakeCache.txt" +if [[ -f "$cache" ]] && ! grep -qx 'CMAKE_GENERATOR:INTERNAL=Ninja' "$cache"; then + die "$build_dir was configured by another generator ($(sed -n 's/^CMAKE_GENERATOR:INTERNAL=//p' "$cache")); remove that directory and rerun" +fi + if [[ "$(uname -s)" == Darwin ]]; then - export CC="$(xcrun --find clang)" - export CXX="$(xcrun --find clang++)" + CC=$(xcrun --find clang) + CXX=$(xcrun --find clang++) + export CC CXX export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-15.0}" leak_detection=0 else leak_detection=1 fi -cmake -S . -B "$build_dir" -DNETGRAPH_CORE_BUILD_TESTS=ON \ - -DNETGRAPH_CORE_SANITIZE="$sanitize" -DCMAKE_BUILD_TYPE="$build_type" -cmake --build "$build_dir" --config "$build_type" --parallel "$jobs" if [[ "$sanitize" == ON ]]; then - export ASAN_OPTIONS="${ASAN_OPTIONS:-detect_leaks=$leak_detection}" + # A sanitizer report must fail, including during test discovery at build time. Append + # the failure settings after the caller's options; keep the platform's leak policy. + export ASAN_OPTIONS="${ASAN_OPTIONS:-detect_leaks=$leak_detection}:halt_on_error=1:exitcode=1" + export UBSAN_OPTIONS="${UBSAN_OPTIONS:+$UBSAN_OPTIONS:}halt_on_error=1:exitcode=1" +fi +if [[ "$tsan" == ON ]]; then + export TSAN_OPTIONS="${TSAN_OPTIONS:-second_deadlock_stack=1}:halt_on_error=1:exitcode=1" +fi +cmake -S . -B "$build_dir" -G Ninja -DCMAKE_BUILD_TYPE="$build_type" \ + -DNETGRAPH_CORE_BUILD_TESTS=ON -DNETGRAPH_CORE_PRIVATE_HOOKS="$hooks" \ + -DNETGRAPH_CORE_SANITIZE="$sanitize" -DNETGRAPH_CORE_TSAN="$tsan" \ + -DNETGRAPH_CORE_COVERAGE="$coverage" -DNETGRAPH_CORE_NATIVE=OFF +cmake --build "$build_dir" --parallel "$jobs" +if [[ "$coverage" == ON ]]; then + # Test discovery runs each binary at build time and writes profiles that are no test's coverage. + find "$build_dir" -name '*.gcda' -delete +fi +log="$build_dir/ctest.log" +ctest --test-dir "$build_dir" --output-on-failure --parallel "$jobs" --timeout 120 --no-tests=error \ + 2>&1 | tee "$log" + +# ctest reports a GTEST_SKIP as "***Skipped" (gtest_discover_tests' skip pattern). +skipped=$(sed -n 's/^ *[0-9]*\/[0-9]* Test *#[0-9]*: \([^ ]*\) .*\*\*\*Skipped.*/\1/p' "$log" | LC_ALL=C sort) +if [[ "$hooks" == OFF ]]; then + expected=$(echo "$EXPECTED_OFF_SKIPS" | sed '/^$/d' | LC_ALL=C sort) + if [[ "$skipped" != "$expected" ]]; then + diff <(echo "$expected") <(echo "$skipped") >&2 || true + die "the skipped tests differ from the expected hook-only tests (< expected, > skipped)" + fi + echo "hooks off: exactly the $(echo "$expected" | wc -l | tr -d ' ') expected hook-only tests skipped" +elif [[ -n "$skipped" ]]; then + die "tests skipped in a hooks-on build: $(echo "$skipped" | tr '\n' ' ')" fi -ctest --test-dir "$build_dir" --build-config "$build_type" \ - --output-on-failure --parallel "$jobs" --timeout 120 --no-tests=error diff --git a/dev/perf/README.md b/dev/perf/README.md index d9c6cc0..ec79894 100644 --- a/dev/perf/README.md +++ b/dev/perf/README.md @@ -1,21 +1,427 @@ -# Performance tooling - -Developer-only tools for measuring NetGraph-Core. Nothing here ships in the -wheel or the sdist, and nothing here is run by CI; the repository gates -(`make check-ci`, `make sanitize-test`) remain the acceptance tests. - -- [`spf_queue/`](spf_queue/): the shortest-path frontier queue and workspace - work (plan, parity harness, old-versus-new bit-gate, NetGraph-level - benchmark, recorded A/B/A results and the independent review). Start with - [`spf_queue/PLAN.md`](spf_queue/PLAN.md). -- [`../benchmark_profiling_overhead.py`](../benchmark_profiling_overhead.py): - SPF and placement micro-benchmark used for the PGO training run - (`make install-pgo`) and for measuring profiling overhead. - -Conventions for performance claims (see `AGENTS.md`): verify correctness -first (bit-identical outputs where the contract is exact, certificates for -flows), then measure old/new/old in fresh processes on one quiet machine, -record load, hashes and raw samples, and treat differences inside the -run-to-run band as inconclusive. The research record behind the queue work, -including the Apple GPU investigation that preceded it, is in -[`../research/`](../research/). +# Timing tool (A/B/A) + +Developer-only tool for timing two builds of NetGraph-Core against each other, as `AGENTS.md` +asks for every performance claim. Nothing here ships in the wheel or the sdist, and no timing +runs in CI; only the tool's own tests do (below), and the repository gates (`make check-ci`, +`make sanitize-test`, `make tsan-test`) stay the acceptance tests. Verify correctness first +(bit-identical results where the library is exact, certificates for flows); time second. + +**A/B/A** means: the reference build A, the candidate build B, then A again, each block in a +fresh process, repeated seven times; the candidate's time is compared with the mean of the two +reference blocks around it, so slow drift of the machine cancels out. A bootstrap interval over +the seven triples says whether a difference is real. + +| File | Role | +| --- | --- | +| `build.sh` | builds one harness binary against the Core of a revision or of the working tree, with its build record | +| `harness.cpp`, `aba_rows.hpp`, `aba_api.hpp` | the harness: the rows (workloads, semantic work per iteration, digests) and the adapter that maps them onto the current public Core API (plus two private functions of the queue rule, below) | +| `aba.py` | runs one A/B/A comparison: blocks, statistics, verdicts, `report.json`, `summary.md`, `blocks.csv`, `meta.txt` | +| `queries/` | the frozen row inputs (`*.json`, `storage/inputs.json`) with their `SHA256SUMS` | +| `test_tool.py` | the tool's own tests (`make perf-tool-test`) | + +## Quick start + +Compare the working tree (candidate) with a revision (reference); both binaries carry the same +rows, each compiled against its own Core. Run on a quiet machine, nothing else running. + +```sh +bash dev/perf/build.sh build/perf/aba/ref # e.g. HEAD, or main +bash dev/perf/build.sh worktree build/perf/aba/cand +venv/bin/python dev/perf/aba.py \ + --old-bin build/perf/aba/ref/harness --new-bin build/perf/aba/cand/harness \ + --old-label ref --new-label cand --rows "kernel.*,maxflow.*" --out build/perf/aba/run +``` + +`aba.py` needs Python 3.10 or newer (the venv's; macOS's system `python3` 3.9 is refused with a +message). The run prints a plan (the expected duration of every row) before the first timed +block, then one line per row, and writes `report.json`, `summary.md` (the table to read), +`blocks.csv` (every block; in an aborted run, those of the finished warm-up pairs and triples) +and `meta.txt` (every hash) to `--out` (default `dev/perf/results//`, ignored by +git). The run's verdict is the worst row verdict, and a row whose results differ between the two +builds fails: the tool compares two builds of the current Core, so a change it times (an +optimisation) must leave every result identical. A change that is meant to change results is +checked for correctness by the test suites, not timed against the old results. The adapter +`aba_api.hpp` is written against the current public API, apart from the queue check of the +storage search rows, which reads the library's queue rule through the private +`detail::bucket_width` and `detail::uses_buckets`: a reference revision with another API needs +the adapter adapted first. + +**Across an API change.** To compare with a reference revision whose API this adapter does not +compile against, build the reference harness in a checkout of that revision instead, with its +own driver adapted to its API and the same rows (`bash dev/perf/build.sh worktree OUT_DIR` in that +checkout), move its `harness.json` aside so that the run does not refuse it for carrying another +driver, and pass the binary as `--old-bin`. The run still requires both harnesses to list the same +rows, record for record, and `report.json` records that side's build record as empty. + +**Exit statuses and acceptance.** `aba.py` exits 0 when the run completed, whatever its verdict +and conformance; 1 when it aborted after starting blocks; 2 when it refused an input or failed +before the first block; 130 for Ctrl-C before the first block. Every failure writes its reason +to `report.json` and `summary.md`, with verdict `aborted` and conformance false. A reason names +the side, mode and row when it comes from a block. A refusal before row selection writes a +minimal report with no rows. A failed harness, malformed response, changed or unreadable binary, +non-finite derived value, operating-system error, internal exception or Ctrl-C ends the run. +The first Ctrl-C stops the active harness; later interrupts are held while the outputs are +written. An interrupt during final writing records an abort and rewrites both outputs. + +Both files are created before the first block. Each final output is attempted even if another +fails; text that cannot be formed is replaced by a minimal abort record. Complete output text +replaces its file in one operation; if directory permissions no longer allow replacement, an +existing writable file is retried directly with the abort reason. An invalid, ambiguous, +occupied or unwritable `--out` cannot receive a new report; existing unrelated files are never +overwritten. A persistent write failure (for example a full disk), or a process killed by an +uncatchable signal, can prevent output. Write failures are also reported on stderr. `--help` +prints help and writes no run files. Both outputs list every selected row with the step it reached: `measured`, and in an +aborted run `not run`, `warm-up` (its warm-up step did not finish: a warm-up block failed, or a +count or the expected duration derived from them is not finite), `planned` (warmed up, not +timed) or `timing` (its timed step did not finish: a timed block failed, or a row statistic +derived from them is not finite). A run killed by another signal writes nothing more. Its exit +status is no acceptance: a run is accepted when it is conforming and its verdict is `pass`, the +first two items of `summary.md` (and `report.json`'s `conforming` and `verdict`). + +## Building the two sides + +`build.sh` requires `bash`, `git`, `tar`, `python3` and the compiler on `PATH` (on macOS, +`xcrun` for an explicitly selected compiler). `vtool` supplies optional build diagnostics. + +`bash dev/perf/build.sh REV OUT_DIR` compiles this checkout's `harness.cpp` (with `aba_rows.hpp` +and `aba_api.hpp`) against the Core sources of `REV`: `worktree` takes this checkout's `include/` +and `src/` with their uncommitted changes; a git revision is extracted with `git archive` into +`OUT_DIR/src`. `REV` is required (there is no default reference), and a revision git does not +know is refused. `OUT_DIR` must be absent or empty; it receives `harness` and the build record +`harness.json`. The compiler is `clang++` from `PATH` with the first `flags=(...)` line of +`build.sh` (`-std=c++20 -O3 -DNDEBUG -funroll-loops -fno-math-errno -fno-trapping-math +-falign-functions=64 -Wall -Wextra`). `CXX` overrides the compiler; on macOS a set `CXX` also gets `-isysroot "$(xcrun +--show-sdk-path)"` (a toolchain compiler such as `$(xcrun --find clang++)` finds no system headers +without it), and a set `CXX` without an SDK is refused. A refusal exits 2. A failed compile exits +with the compiler's status, and a failure while the build record is written exits non-zero; +either leaves neither `harness` nor `harness.json` (a revision's extracted sources stay in +`OUT_DIR/src`, so a second attempt needs another or an emptied `OUT_DIR`). + +Function alignment reduces timing variation due to code placement. Build both sides with +the same flags. The script enables neither `-march=native` nor PGO. These harness flags do not +change the package wheels' Release flags. + +### Build records + +`harness.json` names everything that changes the binary's bytes: + +| Field | Meaning | +| --- | --- | +| `binary`, `binary_sha256` | the output path and the hash of the file written there; the file name is part of the bytes (below) | +| `rev`, `commit`, `dirty_core_paths` | the revision asked for, its commit (`HEAD` for `worktree`), and for `worktree` the number of paths `git status --short` lists under `include/` and `src/` (0 for a revision) | +| `core_sources_sha256` | sha256 of `" \n"` lines over every file under the `include/` and `src/` compiled against (the checkout's or the extracted ones), sorted by path; a path with a part that starts with a dot is left out | +| `driver_sha256` | the sha256 of `dev/perf/harness.cpp` and of every `dev/perf/*.hpp`, by path | +| `compiler`, `compiler_path`, `cxx_env` | the compiler's `--version` line, the program the name resolved to on `PATH`, and `CXX` as given (null when unset) | +| `flags`, `isysroot` | the flags passed (`-isysroot ` included when added), and that SDK path (null when none was passed) | +| `sdk.path`, `sdk.version`, `sdk.sdkroot_env`, `sdk.source` | the SDK `xcrun` names (the one the `clang++` shim and `-isysroot` use) and `SDKROOT`, which overrides the shim's choice; path and version null when `xcrun` names no SDK | +| `deployment_target.requested` | `MACOSX_DEPLOYMENT_TARGET` (null when unset: the compiler then targets the SDK's own OS version) | +| `deployment_target.effective_minos`, `binary_sdk`, `platform`, `raw`, `source` | the minimum OS and SDK version the binary carries, read back from its `LC_BUILD_VERSION` load command with `vtool -show-build` (the first value of each when the binary carries several; `raw` has every line) | +| `built_by` | `dev/perf/build.sh` | + +Besides the sources and the flags, the bytes depend on the **deployment target** (the same +sources built with `MACOSX_DEPLOYMENT_TARGET=15.0` and without it differ), the **output file +name** (the linker signs every arm64 binary ad hoc with the file name as the signature's +identifier, and the binary's UUID follows; copying a binary keeps its bytes, the name at link +time is what counts), the **SDK** and the **compiler**. A pinned binary hash is reproducible +only with all of them equal. `aba.py` refuses a binary whose hash differs from its record. Every field in the table is +required and no other key is accepted, including inside `sdk` and `deployment_target`. +All values are strings except `dirty_core_paths` (a finite nonnegative integer), the three +objects `sdk`, `deployment_target`, `driver_sha256`, and these nullable strings: `compiler_path`, `cxx_env`, `isysroot`, `sdk.path`, `sdk.version`, +`sdk.sdkroot_env`, and `deployment_target.requested`, `effective_minos`, `binary_sdk`, `platform`, +`raw`. A null means the value was unavailable or the environment variable was unset. `binary_sha256` and `core_sources_sha256` are 64 lowercase hexadecimal digits; +`driver_sha256` has exactly this checkout's driver paths and matching hashes. Descriptive +strings are recorded as text; the old build path, revision and compiler are not re-executed or +used to locate inputs. + +## Protocol + +**Blocks.** Each block is a fresh process of one side's harness running one row: setup +(untimed), one untimed iteration that fixes the reference digest, then a fixed iteration count +with every iteration timed; the harness reports the median iteration time and `block_s`, its own +wall-clock measurement of the timed loop. Every iteration must return the reference digest (else +the harness exits 3 and the run aborts), which the block line carries as 16 lowercase hexadecimal +digits. A block's whole output is its response, and the run +aborts unless it is exactly one block line for the requested row ended by a newline, with +nothing before or after it (a usable line is never picked out of a broken response). A block +line is malformed, and aborts the run, when it is not one JSON object, names a key twice or a +key outside the listed fields, or a required number is missing, of another type, not finite (an integer beyond the largest +float counts as not finite) or outside its domain: the iteration count is an integer in [1, 2^63 - 1]; the median iteration +time and the harness's own peak RSS are positive; `block_s`, `setup_s` and the children's peak +RSS may be 0 (the harness prints the two times rounded to microseconds, and starts no child +process); the statistics no rule reads (`mean_s`, `min_s`, `p10_s`, `p90_s`, `max_s`) are +nonnegative when present. A timed block must report the iteration count it was given. The +numeric options are checked the same way (`--tolerance`, `--rss-floor-mib` and +`--calibration-margin` may be 0, `--seed` is an integer of either sign within the finite float range, the others are positive), and the +values derived from them must be finite (`--target-seconds` times one plus the margin, the floor +in bytes). "Inputs" below lists every input's rule. + +**Derived values.** Every number of a block line is finite, and every one but the iteration +count is read as a float, but arithmetic on huge ones need not be finite: the mean of two +medians near 1e308 is infinite, and a ratio against it would read 0, a false pass. So a value +derived from the blocks that is not finite (a mean of A1 and A2, a ratio, the statistic, its +interval, the drift, a median, a peak sum, the memory allowance, a calibrated count, an expected +duration or a finish time beyond any date) aborts the run before the row's statistics or verdict +are recorded or printed. + +**Per row.** One warm-up block per side (discarded; it calibrates the iteration counts), then +seven triples A1, B, A2 in that order (fewer only with `--allow-short`, which labels the run +"not gate evidence"). The counts are **per side**: `ceil(target * (1 + margin) / median_side)` +from that side's own warm-up block (a calibrated count above 2^63 - 1 aborts with the side and row named) (`--target-seconds`, default 2, `--calibration-margin`, +default 0.10), so every block on either side lasts about the target even when the two sides' +speeds differ by orders of magnitude. A1 and A2 share side A's count. `--iters` pins the counts +instead (below). The block value is the median iteration time, so the ratio does not depend on +the counts. The warm-up blocks of every selected row run first; the expected duration of every +row and of the run is printed and stored (`report.json` `plan`, each row's `expected`) before +the first timed block. + +**Statistics.** `r_t = B_t / mean(A1_t, A2_t)`; the statistic is `median_t(r_t - 1)`; its 95 +percent percentile interval comes from `--resamples` (default 10,000) bootstrap resamples of +whole triples, seeded with `--seed`; drift is `median_t |A1_t / A2_t - 1|`. + +**Verdicts per row.** + +* timing: `inconclusive` when the drift exceeds the tolerance (`--tolerance`, default 0.03), + else `pass` when the interval's upper bound is within the tolerance, else `fail`; +* blocks: `fail` when any timed block's `block_s` is below `--target-seconds`; +* same-side clusters: per side, the timed blocks' per-iteration times sorted; a gap between two + consecutive blocks above the tolerance with at least two blocks on each side marks the side + `bimodal` and fails the row even when the median passes; +* memory (below); +* digests: every timed block of both sides must report one and the same digest. + +The row is `fail` if any check fails, else `inconclusive` if timing is, else `pass`. The run's +verdict is `aborted` when the run aborted, else the worst row verdict. A run is **conforming** +(gate evidence) only when it did not abort, `--allow-short` was not given (without it fewer than +seven triples or a target below 2 s are refused; a measured block shorter than the target fails +its row), no timed block was short and the +allocator environment is normal; the summary's first line says which. + +**Memory.** Each block's peak RSS (resident set size: the memory the process holds in RAM) of the +harness process tree is the larger of the harness's own `getrusage` (self + children) +`ru_maxrss` and `ru_maxrss` from the orchestrator's `os.wait4` (macOS reports bytes, Linux KiB, +converted). The harness's own record of the iteration times is the same in every block, whatever +its iteration count: allocated and written before the row's setup, it keeps a count per duration +below 65,536 clock ticks (65.5 µs with the nanosecond clocks of macOS and Linux) and a list of +the longer ones with 65,536 slots, which grows only past 4.3 s of such iterations in one block +(1 MiB of bookkeeping in all; its statistics are those of the sorted list of every iteration's +time). A faster side runs more iterations, and a record that grew with them would count in its +peak. Reference = the largest A1 block peak, candidate = the largest B block peak (over the +timed triples); the row fails when the candidate exceeds `max(1.10 * reference, reference + 8 +MiB)` bytes (`--budget`, `--rss-floor-mib`). A1, the first reference block of every triple, is +fixed in advance: a larger A2 peak or a larger warm-up peak never rescues a failure. Why the +maxima: macOS's allocator keeps freed large blocks in a cache whose pages stay resident, so on +several rows the block peaks of any build land on one of two levels about 9 to 10 MiB apart at +random; medians can fall on different levels by chance and fail identical binaries, while the +largest peaks of both sides reach the upper level. A row records `rss_a1_max_bytes`, +`rss_b_max_bytes`, `rss_allowance_bytes`, `rss_balanced_ratio`, `rss_a2_max_bytes` (a +diagnostic) and `rss_verdict`. The check compares the largest observed peaks; it is no absolute +memory bound. + +**Allocator environment.** `aba.py` neither sets nor clears an allocator variable +(`MallocLargeCache=0` would hide the cache levels and with them a candidate's genuine peak). It +records the harness's environment in `report.json` `allocator_env` and on the summary's +"allocator environment" line: whether `MallocLargeCache` is set, and every variable whose name +starts with `Malloc` or `MALLOC_` or is `DYLD_INSERT_LIBRARIES`, `LD_PRELOAD` or +`GLIBC_TUNABLES`. A run with any such variable is not conforming. + +**Provenance.** `meta.txt` lists the sha256 of both harness binaries (each checked against its +build record when it has one) and of the orchestrator and driver sources (`aba.py`, +`harness.cpp`, `aba_rows.hpp`, `aba_api.hpp`, hashed at the start). A side's binary is +re-hashed before and after every block and at the end of the run; a difference, or a binary +that can no longer be read, aborts the run. `--expect-meta meta.txt` refuses to start unless +every hash equals a previous run's (the refusal names the hashes that differ). Both sides' +harnesses run in the repository root with the environment `aba.py` was started in, unchanged: +`aba.py` sets and clears no variable. + +**Machine state.** Each block records the 1-minute load before and after; the summary has a +machine-state line (load averages, top CPU consumers) at the start and the end of the run. + +### The sides + +Every selected row runs as A1 = `--old-bin`, B = `--new-bin`, A2 = `--old-bin` with every rule +above: two builds of the current Core. Both must run this checkout's driver (`build.sh` builds +them so): the two harnesses must list the same rows, record for record, and a side's build +record, when it has one, must carry this checkout's driver as its `driver_sha256`: exactly the +driver files `meta.txt` names (`harness.cpp`, `aba_rows.hpp`, `aba_api.hpp`), each with the +sha256 it has at the start of the run; else the run is refused, so a harness built before the +rows or the adapter changed is not timed. A side without a build record is not checked this way +(`report.json` records its build record as empty). + +### Selecting rows + +`--rows` takes comma-separated globs over row names and groups (`all`, `construct.*`, +`kernel`); spaces around a glob are ignored, and no text means `all`. **A glob that matches no +row or group refuses the run** before any block, with the list of the groups that exist, even +beside globs that match. The report records, per glob, the rows it selected and the rows of its +groups it did not, and per group the rows the whole selection left out. + +Other options: `--iters row=N,...` (both sides) or `row=NA:NB,...` (per side) pins the timed +blocks' iteration counts of a row in place of the calibrated ones (every item names a selected +row, once; a count is at most 2^63 - 1, the largest the harness reads); `--triples` (default 7, +fewer only with `--allow-short`, "Per row" above); `--seed` (default 20261003) and +`--resamples` of the bootstrap; `--old-label` and `--new-label` (defaults `old` and `new`; two +different labels, below); `--out` (absent or empty). + +**Names.** A row's name and its group are 1 to 64 of the characters `a`-`z`, `0`-`9`, `.`, +`_` and `-`; a side's label is 1 to 32 of `a`-`z`, `0`-`9`, `_` and `-` (no dot). They make the +stderr logs' names, `blocks/.